enhance: port two more import steps to be script compatible

also fix save-file! not waiting on async transact before post-save

deps/graph-parser/script/db_import.cljs

@@ -3,11 +3,9 @@
    developing the import feature and for engineers who want to customize
    the import process"
   (:require [clojure.string :as string]
-            [clojure.edn :as edn]
             [datascript.core :as d]
             ["path" :as node-path]
             ["os" :as os]
-            ["fs" :as fs]
             ["fs/promises" :as fsp]
             [nbb.core :as nbb]
             [babashka.cli :as cli]
@@ -28,32 +26,31 @@
 (defn- build-graph-files
   "Given a graph directory, return absolute, allowed file paths and their contents in preparation
    for parsing"
-  [dir* config]
+  [dir*]
   (let [dir (node-path/resolve dir*)]
     (->> (common-graph/get-files dir)
-         (mapv #(hash-map :rpath %))
-         (remove-hidden-files dir config))))
-
-(defn- read-config
-  "Reads repo-specific config from logseq/config.edn"
-  [dir]
-  (let [config-file (str dir "/" common-config/app-name "/config.edn")]
-    (if (fs/existsSync config-file)
-      (-> config-file fs/readFileSync str edn/read-string)
-      {})))
+         (mapv #(hash-map :rpath %)))))
 
 (defn- import-file-graph-to-db [file-graph-dir conn user-options]
-  (let [config (read-config file-graph-dir)
-        import-options (gp-exporter/setup-import-options
-                        @conn
-                        config
-                        user-options
-                        {:notify-user prn})
-        ;; TODO: Remove logseq/ filter when higher-level import fn is available
-        files (remove #(re-find #"logseq/" (:rpath %)) (build-graph-files file-graph-dir config))]
+  (p/let [*files (build-graph-files file-graph-dir)
+          config-file (first (filter #(string/ends-with? (:rpath %) "logseq/config.edn") *files))
+          _ (assert config-file "No 'logseq/config.edn' found for file graph dir")
+          <read-file #(p/let [s (fsp/readFile (:rpath %))] (str s))
+          ;; TODO: Add :default-config option
+          config (gp-exporter/import-config-file! conn config-file <read-file {:notify-user prn})
+          files (remove-hidden-files file-graph-dir config *files)
+          import-options (gp-exporter/setup-import-options
+                          @conn
+                          config
+                          user-options
+                          {:notify-user prn})
+          logseq-file? #(string/includes? (:rpath %) "logseq/")
+          doc-files (remove logseq-file? files)
+          logseq-files (filter logseq-file? files)]
     ;; (prn :files (count files) files)
-    (gp-exporter/import-from-doc-files!
-     conn files #(p/let [s (fsp/readFile (:rpath %))] (str s)) import-options)))
+    (p/do!
+     (gp-exporter/import-logseq-files conn logseq-files <read-file {:notify-user prn})
+     (gp-exporter/import-from-doc-files! conn doc-files <read-file import-options))))
 
 (def spec
   "Options spec"

deps/graph-parser/src/logseq/graph_parser/exporter.cljs

@@ -12,6 +12,7 @@
             [logseq.db.frontend.property.type :as db-property-type]
             [logseq.common.util.macro :as macro-util]
             [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db :as ldb]
             [promesa.core :as p]))
 
 (defn- get-pid
@@ -517,8 +518,8 @@
 
 (defn import-from-doc-files!
   [conn *doc-files <read-file {:keys [notify-user set-ui-state]
-                                      :or {set-ui-state (constantly nil) notify-user prn}
-                                      :as import-options}]
+                               :or {set-ui-state (constantly nil) notify-user prn}
+                               :as import-options}]
   (set-ui-state [:graph/importing-state :total] (count *doc-files))
   (let [doc-files (mapv #(assoc %1 :idx %2) *doc-files (range 0 (count *doc-files)))]
     (-> (p/loop [_file-map (import-doc-file (get doc-files 0) conn <read-file import-options)
@@ -528,4 +529,40 @@
                      (inc i))))
         (p/catch (fn [e]
                    (notify-user {:msg (str "Import has unexpected error:\n" e)
-                                 :level :error}))))))
+                                 :level :error}))))))
+
+(defn- default-save-file [conn path content]
+  (ldb/transact! conn [{:file/path path
+                        :file/content content
+                        :file/last-modified-at (js/Date.)}]))
+
+(defn import-logseq-files
+  [repo-or-conn logseq-files <read-file {:keys [<save-file notify-user]
+                                         :or {<save-file default-save-file}}]
+  (let [custom-css (first (filter #(string/ends-with? (:rpath %) "logseq/custom.css") logseq-files))
+        custom-js (first (filter #(string/ends-with? (:rpath %) "logseq/custom.js") logseq-files))]
+    (-> (p/do!
+         (when custom-css
+           (-> (<read-file custom-css)
+               (p/then #(<save-file repo-or-conn "logseq/custom.css" %))))
+         (when custom-js
+           (-> (<read-file custom-js)
+               (p/then #(<save-file repo-or-conn "logseq/custom.js" %)))))
+        (p/catch (fn [error]
+                   (notify-user {:msg (str "Import unexpectedly failed while reading logseq files:\n" error)
+                                 :level :error}))))))
+
+(defn import-config-file!
+  [repo-or-conn config-file <read-file {:keys [<save-file notify-user default-config]
+                                        :or {default-config {}
+                                             <save-file default-save-file}}]
+  (-> (<read-file config-file)
+      (p/then #(p/do!
+                (<save-file repo-or-conn "logseq/config.edn" %)
+                ;; Return original config as import process depends on original config e.g. :hidden
+                (edn/read-string %)))
+      (p/catch (fn [err]
+                 (notify-user {:msg "Import may have mistakes due to an invalid config.edn. Recommend re-importing with a valid config.edn"
+                               :level :error
+                               :ex-data {:error err}})
+                 (edn/read-string default-config)))))

src/main/frontend/components/imports.cljs

@@ -186,34 +186,6 @@
           (recur))
         true))))
 
-(defn- import-logseq-files
-  [logseq-files]
-  (let [custom-css (first (filter #(= (:rpath %) "logseq/custom.css") logseq-files))
-        custom-js (first (filter #(= (:rpath %) "logseq/custom.js") logseq-files))]
-    (p/do!
-     (some-> (:file-object custom-css)
-             (.text)
-             (p/then #(db-editor-handler/save-file! "logseq/custom.css" %)))
-     (some-> (:file-object custom-js)
-             (.text)
-             (p/then #(db-editor-handler/save-file! "logseq/custom.js" %))))))
-
-(defn- import-config-file!
-  [{:keys [file-object]}]
-  (-> (.text file-object)
-      (p/then (fn [content]
-                (let [migrated-content (repo-handler/migrate-db-config content)]
-                  (p/do!
-                   (db-editor-handler/save-file! "logseq/config.edn" migrated-content))
-                  ;; Return original config as import process depends on original config e.g. :hidden
-                  (edn/read-string content))))
-      (p/catch (fn [err]
-                 (log/error :import-config-file err)
-                 (notification/show! "Import may have mistakes due to an invalid config.edn. Recommend re-importing with a valid config.edn"
-                                     :warning
-                                     false)
-                 (edn/read-string config/config-default-content)))))
-
 (defn- build-hidden-favorites-page-blocks
   [page-block-uuid-coll]
   (map
@@ -357,6 +329,15 @@
                             :warning false))
       (log/info :import-valid {:msg "Valid import!"
                                :counts (assoc (counts-from-entities entities) :datoms datom-count)}))))
+
+(defn- show-notification [{:keys [msg level ex-data]}]
+  (if (= :error level)
+    (do
+      (notification/show! msg :error)
+      (when ex-data
+        (log/error :import-error ex-data)))
+    (notification/show! msg :warning false)))
+
 (defn- import-file-graph
   [*files {:keys [graph-name tag-classes property-classes]} config-file]
   (state/set-state! :graph/importing :file-graph)
@@ -366,7 +347,14 @@
           _ (async/<! (p->c (repo-handler/new-db! graph-name {:file-graph-import? true})))
           repo (state/get-current-repo)
           db-conn (db/get-db repo false)
-          config (async/<! (p->c (import-config-file! config-file)))
+          <read-file (fn [file] (.text (:file-object file)))
+          config (async/<! (p->c (gp-exporter/import-config-file!
+                                  repo config-file <read-file
+                                  {:notify-user show-notification
+                                   :default-config config/config-default-content
+                                   :<save-file (fn [_ path content]
+                                                 (let [migrated-content (repo-handler/migrate-db-config content)]
+                                                   (db-editor-handler/save-file! path migrated-content)))})))
           files (common-config/remove-hidden-files *files config :rpath)
           logseq-file? #(string/starts-with? (:rpath %) "logseq/")
           doc-files (->> files
@@ -379,21 +367,20 @@
                            config
                            {:tag-classes (set (string/split tag-classes #",\s*"))
                             :property-classes (set (string/split property-classes #",\s*"))}
-                           {:macros (state/get-macros)
-                            :notify-user #(if (= :error (:level %))
-                                            (do
-                                              (notification/show! (:msg %) :error)
-                                              (when (:ex-data %)
-                                                (log/error :import-error (:ex-data %))))
-                                            (notification/show! (:msg %) :warning false))})
+                           {:macros (:macros config)
+                            :notify-user show-notification})
                           {:set-ui-state state/set-state!
                            ;; Write to frontend first as writing to worker first is poor ux with slow streaming changes
                            :import-file (fn import-file [conn m opts]
                                           (let [tx-report
-                                                  (gp-exporter/add-file-to-db-graph conn (:file/path m) (:file/content m) opts)]
-                                              (db-browser/transact! @db-browser/*worker repo (:tx-data tx-report) (:tx-meta tx-report))))})
-          <read-file (fn [file] (.text (:file-object file)))]
-      (async/<! (p->c (import-logseq-files (filter logseq-file? files))))
+                                                (gp-exporter/add-file-to-db-graph conn (:file/path m) (:file/content m) opts)]
+                                            (db-browser/transact! @db-browser/*worker repo (:tx-data tx-report) (:tx-meta tx-report))))})]
+      (async/<! (p->c (gp-exporter/import-logseq-files (state/get-current-repo)
+                                                       (filter logseq-file? files)
+                                                       <read-file
+                                                       {:<save-file (fn [_ path content]
+                                                                      (db-editor-handler/save-file! path content))
+                                                        :notify-user show-notification})))
       (async/<! (import-from-asset-files! asset-files))
       (async/<! (p->c (gp-exporter/import-from-doc-files! db-conn doc-files <read-file import-options)))
       (async/<! (p->c (import-favorites-from-config-edn! db-conn repo config-file)))

src/main/frontend/handler/db_based/editor.cljs

@@ -110,15 +110,16 @@
                       true)]
 
     (when file-valid?
-      (db/transact! [{:file/path path
-                      :file/content content
-                      :file/last-modified-at (js/Date.)}])
+      (p/do!
+       (db/transact! [{:file/path path
+                       :file/content content
+                       :file/last-modified-at (js/Date.)}])
       ;; Post save
-      (cond (= path "logseq/config.edn")
-            (p/let [_ (repo-config-handler/restore-repo-config! (state/get-current-repo) content)]
-              (state/pub-event! [:shortcut/refresh]))
-            (= path "logseq/custom.css")
-            (ui-handler/add-style-if-exists!)))))
+       (cond (= path "logseq/config.edn")
+             (p/let [_ (repo-config-handler/restore-repo-config! (state/get-current-repo) content)]
+               (state/pub-event! [:shortcut/refresh]))
+             (= path "logseq/custom.css")
+             (ui-handler/add-style-if-exists!))))))
 
 (defn- set-heading-aux!
   [block-id heading]