mirror of https://github.com/logseq/logseq
enhance: port two more import steps to be script compatible
also fix save-file! not waiting on async transact before post-savepull/11102/head
parent
94ce77e509
commit
13439ed699
|
@ -3,11 +3,9 @@
|
||||||
developing the import feature and for engineers who want to customize
|
developing the import feature and for engineers who want to customize
|
||||||
the import process"
|
the import process"
|
||||||
(:require [clojure.string :as string]
|
(:require [clojure.string :as string]
|
||||||
[clojure.edn :as edn]
|
|
||||||
[datascript.core :as d]
|
[datascript.core :as d]
|
||||||
["path" :as node-path]
|
["path" :as node-path]
|
||||||
["os" :as os]
|
["os" :as os]
|
||||||
["fs" :as fs]
|
|
||||||
["fs/promises" :as fsp]
|
["fs/promises" :as fsp]
|
||||||
[nbb.core :as nbb]
|
[nbb.core :as nbb]
|
||||||
[babashka.cli :as cli]
|
[babashka.cli :as cli]
|
||||||
|
@ -28,32 +26,31 @@
|
||||||
(defn- build-graph-files
|
(defn- build-graph-files
|
||||||
"Given a graph directory, return absolute, allowed file paths and their contents in preparation
|
"Given a graph directory, return absolute, allowed file paths and their contents in preparation
|
||||||
for parsing"
|
for parsing"
|
||||||
[dir* config]
|
[dir*]
|
||||||
(let [dir (node-path/resolve dir*)]
|
(let [dir (node-path/resolve dir*)]
|
||||||
(->> (common-graph/get-files dir)
|
(->> (common-graph/get-files dir)
|
||||||
(mapv #(hash-map :rpath %))
|
(mapv #(hash-map :rpath %)))))
|
||||||
(remove-hidden-files dir config))))
|
|
||||||
|
|
||||||
(defn- read-config
|
|
||||||
"Reads repo-specific config from logseq/config.edn"
|
|
||||||
[dir]
|
|
||||||
(let [config-file (str dir "/" common-config/app-name "/config.edn")]
|
|
||||||
(if (fs/existsSync config-file)
|
|
||||||
(-> config-file fs/readFileSync str edn/read-string)
|
|
||||||
{})))
|
|
||||||
|
|
||||||
(defn- import-file-graph-to-db [file-graph-dir conn user-options]
|
(defn- import-file-graph-to-db [file-graph-dir conn user-options]
|
||||||
(let [config (read-config file-graph-dir)
|
(p/let [*files (build-graph-files file-graph-dir)
|
||||||
|
config-file (first (filter #(string/ends-with? (:rpath %) "logseq/config.edn") *files))
|
||||||
|
_ (assert config-file "No 'logseq/config.edn' found for file graph dir")
|
||||||
|
<read-file #(p/let [s (fsp/readFile (:rpath %))] (str s))
|
||||||
|
;; TODO: Add :default-config option
|
||||||
|
config (gp-exporter/import-config-file! conn config-file <read-file {:notify-user prn})
|
||||||
|
files (remove-hidden-files file-graph-dir config *files)
|
||||||
import-options (gp-exporter/setup-import-options
|
import-options (gp-exporter/setup-import-options
|
||||||
@conn
|
@conn
|
||||||
config
|
config
|
||||||
user-options
|
user-options
|
||||||
{:notify-user prn})
|
{:notify-user prn})
|
||||||
;; TODO: Remove logseq/ filter when higher-level import fn is available
|
logseq-file? #(string/includes? (:rpath %) "logseq/")
|
||||||
files (remove #(re-find #"logseq/" (:rpath %)) (build-graph-files file-graph-dir config))]
|
doc-files (remove logseq-file? files)
|
||||||
|
logseq-files (filter logseq-file? files)]
|
||||||
;; (prn :files (count files) files)
|
;; (prn :files (count files) files)
|
||||||
(gp-exporter/import-from-doc-files!
|
(p/do!
|
||||||
conn files #(p/let [s (fsp/readFile (:rpath %))] (str s)) import-options)))
|
(gp-exporter/import-logseq-files conn logseq-files <read-file {:notify-user prn})
|
||||||
|
(gp-exporter/import-from-doc-files! conn doc-files <read-file import-options))))
|
||||||
|
|
||||||
(def spec
|
(def spec
|
||||||
"Options spec"
|
"Options spec"
|
||||||
|
|
|
@ -12,6 +12,7 @@
|
||||||
[logseq.db.frontend.property.type :as db-property-type]
|
[logseq.db.frontend.property.type :as db-property-type]
|
||||||
[logseq.common.util.macro :as macro-util]
|
[logseq.common.util.macro :as macro-util]
|
||||||
[logseq.db.sqlite.util :as sqlite-util]
|
[logseq.db.sqlite.util :as sqlite-util]
|
||||||
|
[logseq.db :as ldb]
|
||||||
[promesa.core :as p]))
|
[promesa.core :as p]))
|
||||||
|
|
||||||
(defn- get-pid
|
(defn- get-pid
|
||||||
|
@ -529,3 +530,39 @@
|
||||||
(p/catch (fn [e]
|
(p/catch (fn [e]
|
||||||
(notify-user {:msg (str "Import has unexpected error:\n" e)
|
(notify-user {:msg (str "Import has unexpected error:\n" e)
|
||||||
:level :error}))))))
|
:level :error}))))))
|
||||||
|
|
||||||
|
(defn- default-save-file [conn path content]
|
||||||
|
(ldb/transact! conn [{:file/path path
|
||||||
|
:file/content content
|
||||||
|
:file/last-modified-at (js/Date.)}]))
|
||||||
|
|
||||||
|
(defn import-logseq-files
|
||||||
|
[repo-or-conn logseq-files <read-file {:keys [<save-file notify-user]
|
||||||
|
:or {<save-file default-save-file}}]
|
||||||
|
(let [custom-css (first (filter #(string/ends-with? (:rpath %) "logseq/custom.css") logseq-files))
|
||||||
|
custom-js (first (filter #(string/ends-with? (:rpath %) "logseq/custom.js") logseq-files))]
|
||||||
|
(-> (p/do!
|
||||||
|
(when custom-css
|
||||||
|
(-> (<read-file custom-css)
|
||||||
|
(p/then #(<save-file repo-or-conn "logseq/custom.css" %))))
|
||||||
|
(when custom-js
|
||||||
|
(-> (<read-file custom-js)
|
||||||
|
(p/then #(<save-file repo-or-conn "logseq/custom.js" %)))))
|
||||||
|
(p/catch (fn [error]
|
||||||
|
(notify-user {:msg (str "Import unexpectedly failed while reading logseq files:\n" error)
|
||||||
|
:level :error}))))))
|
||||||
|
|
||||||
|
(defn import-config-file!
|
||||||
|
[repo-or-conn config-file <read-file {:keys [<save-file notify-user default-config]
|
||||||
|
:or {default-config {}
|
||||||
|
<save-file default-save-file}}]
|
||||||
|
(-> (<read-file config-file)
|
||||||
|
(p/then #(p/do!
|
||||||
|
(<save-file repo-or-conn "logseq/config.edn" %)
|
||||||
|
;; Return original config as import process depends on original config e.g. :hidden
|
||||||
|
(edn/read-string %)))
|
||||||
|
(p/catch (fn [err]
|
||||||
|
(notify-user {:msg "Import may have mistakes due to an invalid config.edn. Recommend re-importing with a valid config.edn"
|
||||||
|
:level :error
|
||||||
|
:ex-data {:error err}})
|
||||||
|
(edn/read-string default-config)))))
|
||||||
|
|
|
@ -186,34 +186,6 @@
|
||||||
(recur))
|
(recur))
|
||||||
true))))
|
true))))
|
||||||
|
|
||||||
(defn- import-logseq-files
|
|
||||||
[logseq-files]
|
|
||||||
(let [custom-css (first (filter #(= (:rpath %) "logseq/custom.css") logseq-files))
|
|
||||||
custom-js (first (filter #(= (:rpath %) "logseq/custom.js") logseq-files))]
|
|
||||||
(p/do!
|
|
||||||
(some-> (:file-object custom-css)
|
|
||||||
(.text)
|
|
||||||
(p/then #(db-editor-handler/save-file! "logseq/custom.css" %)))
|
|
||||||
(some-> (:file-object custom-js)
|
|
||||||
(.text)
|
|
||||||
(p/then #(db-editor-handler/save-file! "logseq/custom.js" %))))))
|
|
||||||
|
|
||||||
(defn- import-config-file!
|
|
||||||
[{:keys [file-object]}]
|
|
||||||
(-> (.text file-object)
|
|
||||||
(p/then (fn [content]
|
|
||||||
(let [migrated-content (repo-handler/migrate-db-config content)]
|
|
||||||
(p/do!
|
|
||||||
(db-editor-handler/save-file! "logseq/config.edn" migrated-content))
|
|
||||||
;; Return original config as import process depends on original config e.g. :hidden
|
|
||||||
(edn/read-string content))))
|
|
||||||
(p/catch (fn [err]
|
|
||||||
(log/error :import-config-file err)
|
|
||||||
(notification/show! "Import may have mistakes due to an invalid config.edn. Recommend re-importing with a valid config.edn"
|
|
||||||
:warning
|
|
||||||
false)
|
|
||||||
(edn/read-string config/config-default-content)))))
|
|
||||||
|
|
||||||
(defn- build-hidden-favorites-page-blocks
|
(defn- build-hidden-favorites-page-blocks
|
||||||
[page-block-uuid-coll]
|
[page-block-uuid-coll]
|
||||||
(map
|
(map
|
||||||
|
@ -357,6 +329,15 @@
|
||||||
:warning false))
|
:warning false))
|
||||||
(log/info :import-valid {:msg "Valid import!"
|
(log/info :import-valid {:msg "Valid import!"
|
||||||
:counts (assoc (counts-from-entities entities) :datoms datom-count)}))))
|
:counts (assoc (counts-from-entities entities) :datoms datom-count)}))))
|
||||||
|
|
||||||
|
(defn- show-notification [{:keys [msg level ex-data]}]
|
||||||
|
(if (= :error level)
|
||||||
|
(do
|
||||||
|
(notification/show! msg :error)
|
||||||
|
(when ex-data
|
||||||
|
(log/error :import-error ex-data)))
|
||||||
|
(notification/show! msg :warning false)))
|
||||||
|
|
||||||
(defn- import-file-graph
|
(defn- import-file-graph
|
||||||
[*files {:keys [graph-name tag-classes property-classes]} config-file]
|
[*files {:keys [graph-name tag-classes property-classes]} config-file]
|
||||||
(state/set-state! :graph/importing :file-graph)
|
(state/set-state! :graph/importing :file-graph)
|
||||||
|
@ -366,7 +347,14 @@
|
||||||
_ (async/<! (p->c (repo-handler/new-db! graph-name {:file-graph-import? true})))
|
_ (async/<! (p->c (repo-handler/new-db! graph-name {:file-graph-import? true})))
|
||||||
repo (state/get-current-repo)
|
repo (state/get-current-repo)
|
||||||
db-conn (db/get-db repo false)
|
db-conn (db/get-db repo false)
|
||||||
config (async/<! (p->c (import-config-file! config-file)))
|
<read-file (fn [file] (.text (:file-object file)))
|
||||||
|
config (async/<! (p->c (gp-exporter/import-config-file!
|
||||||
|
repo config-file <read-file
|
||||||
|
{:notify-user show-notification
|
||||||
|
:default-config config/config-default-content
|
||||||
|
:<save-file (fn [_ path content]
|
||||||
|
(let [migrated-content (repo-handler/migrate-db-config content)]
|
||||||
|
(db-editor-handler/save-file! path migrated-content)))})))
|
||||||
files (common-config/remove-hidden-files *files config :rpath)
|
files (common-config/remove-hidden-files *files config :rpath)
|
||||||
logseq-file? #(string/starts-with? (:rpath %) "logseq/")
|
logseq-file? #(string/starts-with? (:rpath %) "logseq/")
|
||||||
doc-files (->> files
|
doc-files (->> files
|
||||||
|
@ -379,21 +367,20 @@
|
||||||
config
|
config
|
||||||
{:tag-classes (set (string/split tag-classes #",\s*"))
|
{:tag-classes (set (string/split tag-classes #",\s*"))
|
||||||
:property-classes (set (string/split property-classes #",\s*"))}
|
:property-classes (set (string/split property-classes #",\s*"))}
|
||||||
{:macros (state/get-macros)
|
{:macros (:macros config)
|
||||||
:notify-user #(if (= :error (:level %))
|
:notify-user show-notification})
|
||||||
(do
|
|
||||||
(notification/show! (:msg %) :error)
|
|
||||||
(when (:ex-data %)
|
|
||||||
(log/error :import-error (:ex-data %))))
|
|
||||||
(notification/show! (:msg %) :warning false))})
|
|
||||||
{:set-ui-state state/set-state!
|
{:set-ui-state state/set-state!
|
||||||
;; Write to frontend first as writing to worker first is poor ux with slow streaming changes
|
;; Write to frontend first as writing to worker first is poor ux with slow streaming changes
|
||||||
:import-file (fn import-file [conn m opts]
|
:import-file (fn import-file [conn m opts]
|
||||||
(let [tx-report
|
(let [tx-report
|
||||||
(gp-exporter/add-file-to-db-graph conn (:file/path m) (:file/content m) opts)]
|
(gp-exporter/add-file-to-db-graph conn (:file/path m) (:file/content m) opts)]
|
||||||
(db-browser/transact! @db-browser/*worker repo (:tx-data tx-report) (:tx-meta tx-report))))})
|
(db-browser/transact! @db-browser/*worker repo (:tx-data tx-report) (:tx-meta tx-report))))})]
|
||||||
<read-file (fn [file] (.text (:file-object file)))]
|
(async/<! (p->c (gp-exporter/import-logseq-files (state/get-current-repo)
|
||||||
(async/<! (p->c (import-logseq-files (filter logseq-file? files))))
|
(filter logseq-file? files)
|
||||||
|
<read-file
|
||||||
|
{:<save-file (fn [_ path content]
|
||||||
|
(db-editor-handler/save-file! path content))
|
||||||
|
:notify-user show-notification})))
|
||||||
(async/<! (import-from-asset-files! asset-files))
|
(async/<! (import-from-asset-files! asset-files))
|
||||||
(async/<! (p->c (gp-exporter/import-from-doc-files! db-conn doc-files <read-file import-options)))
|
(async/<! (p->c (gp-exporter/import-from-doc-files! db-conn doc-files <read-file import-options)))
|
||||||
(async/<! (p->c (import-favorites-from-config-edn! db-conn repo config-file)))
|
(async/<! (p->c (import-favorites-from-config-edn! db-conn repo config-file)))
|
||||||
|
|
|
@ -110,6 +110,7 @@
|
||||||
true)]
|
true)]
|
||||||
|
|
||||||
(when file-valid?
|
(when file-valid?
|
||||||
|
(p/do!
|
||||||
(db/transact! [{:file/path path
|
(db/transact! [{:file/path path
|
||||||
:file/content content
|
:file/content content
|
||||||
:file/last-modified-at (js/Date.)}])
|
:file/last-modified-at (js/Date.)}])
|
||||||
|
@ -118,7 +119,7 @@
|
||||||
(p/let [_ (repo-config-handler/restore-repo-config! (state/get-current-repo) content)]
|
(p/let [_ (repo-config-handler/restore-repo-config! (state/get-current-repo) content)]
|
||||||
(state/pub-event! [:shortcut/refresh]))
|
(state/pub-event! [:shortcut/refresh]))
|
||||||
(= path "logseq/custom.css")
|
(= path "logseq/custom.css")
|
||||||
(ui-handler/add-style-if-exists!)))))
|
(ui-handler/add-style-if-exists!))))))
|
||||||
|
|
||||||
(defn- set-heading-aux!
|
(defn- set-heading-aux!
|
||||||
[block-id heading]
|
[block-id heading]
|
||||||
|
|
Loading…
Reference in New Issue