greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: stuck importing from edn/json 

Also, this commit added the progress UI for importing and removed the
:file/writes core.async channel.
pull/6486/head
Tienson Qin 2022-08-25 16:45:04 +08:00
parent 6807931ee8
commit f4aa08fbd3
17 changed files with 164 additions and 275 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/frontend/components/export.cljs
View File

@ -12,8 +12,13 @@
(when-let [current-repo (state/get-current-repo)] (when-let [current-repo (state/get-current-repo)]
[:div.export [:div.export
[:h1.title "Export"] [:h1.title "Export"]
[:ul.mr-1 [:ul.mr-1
[:li.mb-4
[:a.font-medium {:on-click #(export/export-repo-as-edn-v2! current-repo)}
(t :export-edn)]]
[:li.mb-4
[:a.font-medium {:on-click #(export/export-repo-as-json-v2! current-repo)}
(t :export-json)]]
(when (util/electron?) (when (util/electron?)
[:li.mb-4 [:li.mb-4
[:a.font-medium {:on-click #(export/export-repo-as-html! current-repo)} [:a.font-medium {:on-click #(export/export-repo-as-html! current-repo)}
@ -25,12 +30,6 @@
[:li.mb-4 [:li.mb-4
[:a.font-medium {:on-click #(export/export-repo-as-opml! current-repo)} [:a.font-medium {:on-click #(export/export-repo-as-opml! current-repo)}
(t :export-opml)]]) (t :export-opml)]])
[:li.mb-4
[:a.font-medium {:on-click #(export/export-repo-as-edn-v2! current-repo)}
(t :export-edn)]]
[:li.mb-4
[:a.font-medium {:on-click #(export/export-repo-as-json-v2! current-repo)}
(t :export-json)]]
(when-not (mobile-util/native-platform?) (when-not (mobile-util/native-platform?)
[:li.mb-4 [:li.mb-4
[:a.font-medium {:on-click #(export/export-repo-as-roam-json! current-repo)} [:a.font-medium {:on-click #(export/export-repo-as-roam-json! current-repo)}

80
src/main/frontend/components/external.cljs
View File

@ -1,80 +0,0 @@
;; deprecated by the onboarding import panel frontend.components.onboarding.setups
(ns frontend.components.external
(:require [rum.core :as rum]
[goog.object :as gobj]
[clojure.string :as string]
[frontend.handler.notification :as notification]
[frontend.handler.external :as external-handler]
[frontend.ui :as ui]
[reitit.frontend.easy :as rfe]))
(defonce *roam-importing? (atom nil))
(defonce *opml-importing? (atom nil))
(defonce *opml-imported-pages (atom nil))
(rum/defc import-cp < rum/reactive
[]
(let [roam-importing? (rum/react *roam-importing?)
opml-importing? (rum/react *opml-importing?)]
[:div#import
[:h1.title "Import JSON from Roam Research"]
[:input
{:id "import-roam"
:type "file"
:on-change (fn [e]
(let [file (first (array-seq (.-files (.-target e))))
file-name (gobj/get file "name")]
(if (string/ends-with? file-name ".json")
(do
(reset! *roam-importing? true)
(let [reader (js/FileReader.)]
(set! (.-onload reader)
(fn [e]
(let [text (.. e -target -result)]
(external-handler/import-from-roam-json! text
#(reset! *roam-importing? false)))))
(.readAsText reader file)))
(notification/show! "Please choose a JSON file."
:error))))}]
[:hr]
[:div.mt-4
(case roam-importing?
true (ui/loading "Loading")
false [:b "Importing finished!"]
nil)]
;;
[:h1.title "Import OPML"]
[:input
{:id "import-opml"
:type "file"
:on-change (fn [e]
(let [file (first (array-seq (.-files (.-target e))))
file-name (gobj/get file "name")]
(if (string/ends-with? file-name ".opml")
(do
(reset! *opml-importing? true)
(let [reader (js/FileReader.)]
(set! (.-onload reader)
(fn [e]
(let [text (.. e -target -result)]
(external-handler/import-from-opml! text
(fn [pages]
(reset! *opml-imported-pages pages)
(reset! *opml-importing? false))))))
(.readAsText reader file)))
(notification/show! "Please choose a OPML file."
:error))))}]
[:div.mt-4
(case opml-importing?
true (ui/loading "Loading")
false [:div
[:b "Importing finished!"]
[:tr
(mapv (fn [page-name] [:tb
[:a {:href (rfe/href :page {:name page-name})} page-name]])
@*opml-imported-pages)]]
nil)]]))

10
src/main/frontend/components/onboarding.cljs
View File

@ -13,8 +13,8 @@
[] []
[:div.help.cp__sidebar-help-docs [:div.help.cp__sidebar-help-docs
(let [discourse-with-icon [:div.flex-row.inline-flex.items-center (let [discourse-with-icon [:div.flex-row.inline-flex.items-center
[:span.mr-1 (t :help/forum-community)] [:span.mr-1 (t :help/forum-community)]
(ui/icon "message-circle" {:style {:font-size 20}})] (ui/icon "message-circle" {:style {:font-size 20}})]
list list
[{:title "Usage" [{:title "Usage"
:children [[[:a :children [[[:a
@ -25,7 +25,7 @@
[(t :help/docs) "https://docs.logseq.com/"] [(t :help/docs) "https://docs.logseq.com/"]
[(t :help/start) "https://docs.logseq.com/#/page/tutorial"] [(t :help/start) "https://docs.logseq.com/#/page/tutorial"]
["FAQ" "https://docs.logseq.com/#/page/faq"]]} ["FAQ" "https://docs.logseq.com/#/page/faq"]]}
{:title "Community" {:title "Community"
:children [[(t :help/awesome-logseq) "https://github.com/logseq/awesome-logseq"] :children [[(t :help/awesome-logseq) "https://github.com/logseq/awesome-logseq"]
[(t :help/blog) "https://blog.logseq.com"] [(t :help/blog) "https://blog.logseq.com"]
@ -36,7 +36,7 @@
[(t :help/bug) "https://github.com/logseq/logseq/issues/new?labels=from:in-app&template=bug_report.yaml"] [(t :help/bug) "https://github.com/logseq/logseq/issues/new?labels=from:in-app&template=bug_report.yaml"]
[(t :help/feature) "https://discuss.logseq.com/c/feature-requests/"] [(t :help/feature) "https://discuss.logseq.com/c/feature-requests/"]
[(t :help/changelog) "https://docs.logseq.com/#/page/changelog"]]} [(t :help/changelog) "https://docs.logseq.com/#/page/changelog"]]}
{:title "About" {:title "About"
:children [[(t :help/about) "https://logseq.com/blog/about"]]} :children [[(t :help/about) "https://logseq.com/blog/about"]]}
@ -44,7 +44,7 @@
:children [[(t :help/privacy) "https://logseq.com/blog/privacy-policy"] :children [[(t :help/privacy) "https://logseq.com/blog/privacy-policy"]
[(t :help/terms) "https://logseq.com/blog/terms"]]}]] [(t :help/terms) "https://logseq.com/blog/terms"]]}]]
(map (fn [sublist] (map (fn [sublist]
[[:p.mt-4.mb-1 [:b (:title sublist)]] [[:p.mt-4.mb-1 [:b (:title sublist)]]

1
src/main/frontend/components/onboarding/index.css
View File

@ -314,7 +314,6 @@ body[data-page=import] {
small { small {
font-size: 11px; font-size: 11px;
text-align: center;
} }
&:hover { &:hover {

106
src/main/frontend/components/onboarding/setups.cljs
View File

@ -2,9 +2,11 @@
(:require [frontend.state :as state] (:require [frontend.state :as state]
[rum.core :as rum] [rum.core :as rum]
[frontend.ui :as ui] [frontend.ui :as ui]
[frontend.context.i18n :refer [t]]
[frontend.components.svg :as svg] [frontend.components.svg :as svg]
[frontend.handler.page :as page-handler] [frontend.handler.page :as page-handler]
[frontend.handler.route :as route-handler] [frontend.handler.route :as route-handler]
[frontend.handler.ui :as ui-handler]
[frontend.util :as util] [frontend.util :as util]
[frontend.handler.web.nfs :as nfs] [frontend.handler.web.nfs :as nfs]
[frontend.mobile.util :as mobile-util] [frontend.mobile.util :as mobile-util]
@ -126,15 +128,13 @@
[:strong.uppercase title] [:strong.uppercase title]
[:small.opacity-50 label]]]))]]]))) [:small.opacity-50 label]]]))]]])))
(defonce *roam-importing? (atom nil))
(defonce *lsq-importing? (atom nil))
(defonce *opml-importing? (atom nil))
(defonce *opml-imported-pages (atom nil)) (defonce *opml-imported-pages (atom nil))
(defn- finished-cb (defn- finished-cb
[] []
(route-handler/redirect-to-home!)
(notification/show! "Import finished!" :success) (notification/show! "Import finished!" :success)
(route-handler/redirect-to-home!)) (ui-handler/re-render-root!))
(defn- roam-import-handler (defn- roam-import-handler
[e] [e]
@ -142,13 +142,16 @@
file-name (gobj/get file "name")] file-name (gobj/get file "name")]
(if (string/ends-with? file-name ".json") (if (string/ends-with? file-name ".json")
(do (do
(reset! *roam-importing? true) (state/set-state! :graph/importing :roam-json)
(let [reader (js/FileReader.)] (let [reader (js/FileReader.)]
(set! (.-onload reader) (set! (.-onload reader)
(fn [e] (fn [e]
(let [text (.. e -target -result)] (let [text (.. e -target -result)]
(external-handler/import-from-roam-json! text (external-handler/import-from-roam-json!
#(do (reset! *roam-importing? false) (finished-cb)))))) text
#(do
(state/set-state! :graph/importing nil)
(finished-cb))))))
(.readAsText reader file))) (.readAsText reader file)))
(notification/show! "Please choose a JSON file." (notification/show! "Please choose a JSON file."
:error)))) :error))))
@ -157,32 +160,27 @@
[e] [e]
(let [file (first (array-seq (.-files (.-target e)))) (let [file (first (array-seq (.-files (.-target e))))
file-name (some-> (gobj/get file "name") file-name (some-> (gobj/get file "name")
(string/lower-case))] (string/lower-case))
(cond (string/ends-with? file-name ".edn") edn? (string/ends-with? file-name ".edn")
(do json? (string/ends-with? file-name ".json")]
(reset! *lsq-importing? true) (if (or edn? json?)
(let [reader (js/FileReader.)] (do
(set! (.-onload reader) (state/set-state! :graph/importing :logseq)
(fn [e] (let [reader (js/FileReader.)
(let [text (.. e -target -result)] import-f (if edn?
(external-handler/import-from-edn! text external-handler/import-from-edn!
#(do (reset! *lsq-importing? false) (finished-cb)))))) external-handler/import-from-json!)]
(.readAsText reader file))) (set! (.-onload reader)
(fn [e]
(string/ends-with? file-name ".json") (let [text (.. e -target -result)]
(do (import-f
(reset! *lsq-importing? true) text
(let [reader (js/FileReader.)] #(do
(set! (.-onload reader) (state/set-state! :graph/importing nil)
(fn [e] (finished-cb))))))
(let [text (.. e -target -result)] (.readAsText reader file)))
(external-handler/import-from-json! text (notification/show! "Please choose an EDN or a JSON file."
#(do (reset! *lsq-importing? false) (finished-cb)))))) :error))))
(.readAsText reader file)))
:else
(notification/show! "Please choose an EDN or a JSON file."
:error))))
(defn- opml-import-handler (defn- opml-import-handler
[e] [e]
@ -190,7 +188,7 @@
file-name (gobj/get file "name")] file-name (gobj/get file "name")]
(if (string/ends-with? file-name ".opml") (if (string/ends-with? file-name ".opml")
(do (do
(reset! *opml-importing? true) (state/set-state! :graph/importing :opml)
(let [reader (js/FileReader.)] (let [reader (js/FileReader.)]
(set! (.-onload reader) (set! (.-onload reader)
(fn [e] (fn [e]
@ -198,7 +196,7 @@
(external-handler/import-from-opml! text (external-handler/import-from-opml! text
(fn [pages] (fn [pages]
(reset! *opml-imported-pages pages) (reset! *opml-imported-pages pages)
(reset! *opml-importing? false) (state/set-state! :graph/importing nil)
(finished-cb)))))) (finished-cb))))))
(.readAsText reader file))) (.readAsText reader file)))
(notification/show! "Please choose a OPML file." (notification/show! "Please choose a OPML file."
@ -206,11 +204,18 @@
(rum/defc importer < rum/reactive (rum/defc importer < rum/reactive
[{:keys [query-params]}] [{:keys [query-params]}]
(let [roam-importing? (rum/react *roam-importing?) (if (state/sub :graph/importing)
lsq-importing? (rum/react *lsq-importing?) (let [{:keys [total current-idx current-page]} (state/sub :graph/importing-state)
opml-importing? (rum/react *opml-importing?) left-label [:div.flex.flex-row.font-bold
importing? (or roam-importing? lsq-importing? opml-importing?)] (t :importing)
[:div.hidden.md:flex.flex-row
[:span.mr-1 ": "]
[:div.text-ellipsis-wrapper {:style {:max-width 300}}
current-page]]]
width (js/Math.round (* (.toFixed (/ current-idx total) 2) 100))
process (when (and total current-idx)
(str current-idx "/" total))]
(ui/progress-bar-with-label width left-label process))
(setups-container (setups-container
:importer :importer
[:article.flex.flex-col.items-center.importer.py-16.px-8 [:article.flex.flex-col.items-center.importer.py-16.px-8
@ -219,39 +224,30 @@
[:h2 "If they are in a JSON, EDN or Markdown format Logseq can work with them."]] [:h2 "If they are in a JSON, EDN or Markdown format Logseq can work with them."]]
[:section.d.md:flex [:section.d.md:flex
[:label.action-input.flex.items-center.mx-2.my-2 [:label.action-input.flex.items-center.mx-2.my-2
{:disabled importing?}
[:span.as-flex-center [:i (svg/roam-research 28)]] [:span.as-flex-center [:i (svg/roam-research 28)]]
[:div.flex.flex-col [:div.flex.flex-col
(if roam-importing? [[:strong "RoamResearch"]
(ui/loading "Importing ...") [:small "Import a JSON Export of your Roam graph"]]]
[[:strong "RoamResearch"]
[:small "Import a JSON Export of your Roam graph"]])]
[:input.absolute.hidden [:input.absolute.hidden
{:id "import-roam" {:id "import-roam"
:type "file" :type "file"
:on-change roam-import-handler}]] :on-change roam-import-handler}]]
[:label.action-input.flex.items-center.mx-2.my-2 [:label.action-input.flex.items-center.mx-2.my-2
{:disabled importing?}
[:span.as-flex-center [:i (svg/logo 28)]] [:span.as-flex-center [:i (svg/logo 28)]]
[:span.flex.flex-col [:span.flex.flex-col
(if lsq-importing? [[:strong "EDN / JSON"]
(ui/loading "Importing ...") [:small "Import an EDN or a JSON Export of your Logseq graph"]]]
[[:strong "EDN / JSON"]
[:small "Import an EDN or a JSON Export of your Logseq graph"]])]
[:input.absolute.hidden [:input.absolute.hidden
{:id "import-lsq" {:id "import-lsq"
:type "file" :type "file"
:on-change lsq-import-handler}]] :on-change lsq-import-handler}]]
[:label.action-input.flex.items-center.mx-2.my-2 [:label.action-input.flex.items-center.mx-2.my-2
{:disabled importing?}
[:span.as-flex-center (ui/icon "sitemap" {:style {:fontSize "26px"}})] [:span.as-flex-center (ui/icon "sitemap" {:style {:fontSize "26px"}})]
[:span.flex.flex-col [:span.flex.flex-col
(if opml-importing? [[:strong "OPML"]
(ui/loading "Importing ...") [:small " Import OPML files"]]]
[[:strong "OPML"]
[:small " Import OPML files"]])]
[:input.absolute.hidden [:input.absolute.hidden
{:id "import-opml" {:id "import-opml"

1
src/main/frontend/dicts.cljc
View File

@ -252,6 +252,7 @@
:developer-mode-alert "You need to restart the app to enable the plugin system. Do you want to restart it now?" :developer-mode-alert "You need to restart the app to enable the plugin system. Do you want to restart it now?"
:relaunch-confirm-to-work "Should relaunch app to make it work. Do you want to restart it now?" :relaunch-confirm-to-work "Should relaunch app to make it work. Do you want to restart it now?"
:import "Import" :import "Import"
:importing "Importing"
:join-community "Join the community" :join-community "Join the community"
:sponsor-us "Sponsor Us" :sponsor-us "Sponsor Us"
:discourse-title "Our forum!" :discourse-title "Our forum!"

6
src/main/frontend/fs/capacitor_fs.cljs
View File

@ -185,15 +185,13 @@
repo-dir (config/get-local-dir repo) repo-dir (config/get-local-dir repo)
ext (util/get-file-ext path) ext (util/get-file-ext path)
db-content (or old-content (db/get-file repo path) "") db-content (or old-content (db/get-file repo path) "")
contents-matched? (contents-matched? disk-content db-content) contents-matched? (contents-matched? disk-content db-content)]
pending-writes (state/get-write-chan-length)]
(cond (cond
(and (and
(not= stat :not-found) ; file on the disk was deleted (not= stat :not-found) ; file on the disk was deleted
(not contents-matched?) (not contents-matched?)
(not (contains? #{"excalidraw" "edn" "css"} ext)) (not (contains? #{"excalidraw" "edn" "css"} ext))
(not (string/includes? path "/.recycle/")) (not (string/includes? path "/.recycle/")))
(zero? pending-writes))
(p/let [disk-content (encrypt/decrypt disk-content)] (p/let [disk-content (encrypt/decrypt disk-content)]
(state/pub-event! [:file/not-matched-from-disk path disk-content content])) (state/pub-event! [:file/not-matched-from-disk path disk-content content]))

4
src/main/frontend/fs/nfs.cljs
View File

@ -163,7 +163,6 @@
(if file-handle (if file-handle
(-> (p/let [local-file (.getFile file-handle) (-> (p/let [local-file (.getFile file-handle)
local-content (.text local-file) local-content (.text local-file)
pending-writes (state/get-write-chan-length)
ext (string/lower-case (util/get-file-ext path)) ext (string/lower-case (util/get-file-ext path))
db-content (db/get-file repo path) db-content (db/get-file repo path)
contents-matched? (contents-matched? local-content (or db-content ""))] contents-matched? (contents-matched? local-content (or db-content ""))]
@ -173,8 +172,7 @@
(not (:skip-compare? opts)) (not (:skip-compare? opts))
(not contents-matched?) (not contents-matched?)
(not (contains? #{"excalidraw" "edn" "css"} ext)) (not (contains? #{"excalidraw" "edn" "css"} ext))
(not (string/includes? path "/.recycle/")) (not (string/includes? path "/.recycle/")))
(zero? pending-writes))
(p/let [local-content (encrypt/decrypt local-content)] (p/let [local-content (encrypt/decrypt local-content)]
(state/pub-event! [:file/not-matched-from-disk path local-content content])) (state/pub-event! [:file/not-matched-from-disk path local-content content]))
(p/let [_ (verify-permission repo file-handle true) (p/let [_ (verify-permission repo file-handle true)

6
src/main/frontend/fs/node.cljs
View File

@ -53,15 +53,13 @@
disk-content (or disk-content "") disk-content (or disk-content "")
ext (string/lower-case (util/get-file-ext path)) ext (string/lower-case (util/get-file-ext path))
db-content (or old-content (db/get-file repo path) "") db-content (or old-content (db/get-file repo path) "")
contents-matched? (contents-matched? disk-content db-content) contents-matched? (contents-matched? disk-content db-content)]
pending-writes (state/get-write-chan-length)]
(cond (cond
(and (and
(not= stat :not-found) ; file on the disk was deleted (not= stat :not-found) ; file on the disk was deleted
(not contents-matched?) (not contents-matched?)
(not (contains? #{"excalidraw" "edn" "css"} ext)) (not (contains? #{"excalidraw" "edn" "css"} ext))
(not (string/includes? path "/.recycle/")) (not (string/includes? path "/.recycle/")))
(zero? pending-writes))
(p/let [disk-content (encrypt/decrypt disk-content)] (p/let [disk-content (encrypt/decrypt disk-content)]
(state/pub-event! [:file/not-matched-from-disk path disk-content content])) (state/pub-event! [:file/not-matched-from-disk path disk-content content]))

2
src/main/frontend/handler.cljs
View File

@ -217,7 +217,7 @@
(p/do! (mobile-util/hide-splash))) (p/do! (mobile-util/hide-splash)))
(db/run-batch-txs!) (db/run-batch-txs!)
(file-handler/run-writes-chan!)
(when config/dev? (when config/dev?
(enable-datalog-console)) (enable-datalog-console))
(when (util/electron?) (when (util/electron?)

75
src/main/frontend/handler/external.cljs
View File

@ -16,7 +16,9 @@
[frontend.handler.page :as page] [frontend.handler.page :as page]
[frontend.handler.editor :as editor] [frontend.handler.editor :as editor]
[frontend.handler.notification :as notification] [frontend.handler.notification :as notification]
[frontend.util :as util])) [frontend.util :as util]
[clojure.core.async :as async]
[medley.core :as medley]))
(defn index-files! (defn index-files!
"Create file structure, then parse into DB (client only)" "Create file structure, then parse into DB (client only)"
@ -42,7 +44,7 @@
".md")] ".md")]
{:file/path path {:file/path path
:file/content text})))) :file/content text}))))
files) files)
files (remove nil? files)] files (remove nil? files)]
(repo-handler/parse-files-and-load-to-db! repo files nil) (repo-handler/parse-files-and-load-to-db! repo files nil)
(let [files (->> (map (fn [{:file/keys [path content]}] (when path [path content])) files) (let [files (->> (map (fn [{:file/keys [path content]}] (when path [path content])) files)
@ -53,13 +55,13 @@
:finish-handler finish-handler})) :finish-handler finish-handler}))
(let [journal-pages-tx (let [titles (filter date/valid-journal-title? titles)] (let [journal-pages-tx (let [titles (filter date/valid-journal-title? titles)]
(map (map
(fn [title] (fn [title]
(let [day (date/journal-title->int title) (let [day (date/journal-title->int title)
page-name (util/page-name-sanity-lc (date-time-util/int->journal-title day (state/get-date-formatter)))] page-name (util/page-name-sanity-lc (date-time-util/int->journal-title day (state/get-date-formatter)))]
{:block/name page-name {:block/name page-name
:block/journal? true :block/journal? true
:block/journal-day day})) :block/journal-day day}))
titles))] titles))]
(when (seq journal-pages-tx) (when (seq journal-pages-tx)
(db/transact! repo journal-pages-tx))))) (db/transact! repo journal-pages-tx)))))
@ -132,27 +134,44 @@
"\nSkipped and continue the remaining import.") :error)))))) "\nSkipped and continue the remaining import.") :error))))))
title) title)
(defn- pre-transact-uuids (defn- pre-transact-uuids!
"Collect all uuids from page trees and write them to the db before hand." "Collect all uuids from page trees and write them to the db before hand."
[pages] [pages]
(let [uuids (map (fn [block] (let [uuids (mapv (fn [block]
{:block/uuid (:uuid block)}) {:block/uuid (:uuid block)})
(mapcat #(tree-seq map? :children %) (mapcat #(tree-seq map? :children %)
pages))] pages))]
(db/transact! uuids) (db/transact! uuids)))
pages))
(defn- import-from-tree! (defn- import-from-tree!
"Not rely on file system - backend compatible. "Not rely on file system - backend compatible.
tree-translator-fn: translate exported tree structure to the desired tree for import" tree-translator-fn: translate exported tree structure to the desired tree for import"
[data tree-translator-fn] [data tree-translator-fn]
(try (->> (:blocks data) (let [imported-chan (async/promise-chan)
(map tree-translator-fn) blocks (->> (:blocks data)
(pre-transact-uuids) (mapv tree-translator-fn )
(mapv create-page-with-exported-tree!)) (sort-by :title)
(editor/set-blocks-id! (db/get-all-referenced-blocks-uuid)) (medley/indexed))
(catch js/Error e job-chan (async/to-chan! blocks)]
(notification/show! (str "Error happens when importing:\n" e) :error)))) (try
(state/set-state! [:graph/importing-state :total] (count blocks))
(pre-transact-uuids! blocks)
(async/go-loop []
(if-let [[i block] (async/<! job-chan)]
(do
(state/set-state! [:graph/importing-state :current-idx] (inc i))
(state/set-state! [:graph/importing-state :current-page] (:title block))
(async/<! (async/timeout 10))
(create-page-with-exported-tree! block)
(recur))
(do
(editor/set-blocks-id! (db/get-all-referenced-blocks-uuid))
(async/offer! imported-chan true))))
(catch :default e
(notification/show! (str "Error happens when importing:\n" e) :error)
(async/offer! imported-chan true)))
imported-chan))
(defn tree-vec-translate-edn (defn tree-vec-translate-edn
"Actions to do for loading edn tree structure. "Actions to do for loading edn tree structure.
@ -177,8 +196,9 @@
(defn import-from-edn! (defn import-from-edn!
[raw finished-ok-handler] [raw finished-ok-handler]
(import-from-tree! (edn/read-string raw) tree-vec-translate-edn) (async/go
(finished-ok-handler nil)) ;; it was designed to accept a list of imported page names but now deprecated (async/<! (import-from-tree! (edn/read-string raw) tree-vec-translate-edn))
(finished-ok-handler nil))) ;; it was designed to accept a list of imported page names but now deprecated
(defn tree-vec-translate-json (defn tree-vec-translate-json
"Actions to do for loading json tree structure. "Actions to do for loading json tree structure.
@ -210,5 +230,6 @@
[raw finished-ok-handler] [raw finished-ok-handler]
(let [json (js/JSON.parse raw) (let [json (js/JSON.parse raw)
clj-data (js->clj json :keywordize-keys true)] clj-data (js->clj json :keywordize-keys true)]
(import-from-tree! clj-data tree-vec-translate-json)) (async/go
(finished-ok-handler nil)) ;; it was designed to accept a list of imported page names but now deprecated (async/<! (import-from-tree! clj-data tree-vec-translate-json))
(finished-ok-handler nil)))) ;; it was designed to accept a list of imported page names but now deprecated

64
src/main/frontend/handler/file.cljs
View File

@ -2,8 +2,6 @@
(:refer-clojure :exclude [load-file]) (:refer-clojure :exclude [load-file])
(:require ["/frontend/utils" :as utils] (:require ["/frontend/utils" :as utils]
[borkdude.rewrite-edn :as rewrite] [borkdude.rewrite-edn :as rewrite]
[cljs.core.async.interop :refer [<p!]]
[clojure.core.async :as async]
[frontend.config :as config] [frontend.config :as config]
[frontend.db :as db] [frontend.db :as db]
[frontend.fs :as fs] [frontend.fs :as fs]
@ -206,30 +204,8 @@
(alter-file repo path new-content {:reset? false (alter-file repo path new-content {:reset? false
:re-render-root? false})) :re-render-root? false}))
(defn alter-files
[repo files {:keys [reset? update-db?]
:or {reset? false
update-db? true}
:as opts}]
;; old file content
(let [file->content (let [paths (map first files)]
(zipmap paths
(map (fn [path] (db/get-file repo path)) paths)))]
;; update db
(when update-db?
(doseq [[path content] files]
(if reset?
(reset-file! repo path content)
(db/set-file-content! repo path content))))
(when-let [chan (state/get-file-write-chan)]
(let [chan-callback (:chan-callback opts)]
(async/put! chan [repo files opts file->content])
(when chan-callback
(chan-callback))))))
(defn alter-files-handler! (defn alter-files-handler!
[repo files {:keys [finish-handler chan]} file->content] [repo files {:keys [finish-handler]} file->content]
(let [write-file-f (fn [[path content]] (let [write-file-f (fn [[path content]]
(when path (when path
(let [original-content (get file->content path)] (let [original-content (get file->content path)]
@ -254,30 +230,30 @@
:error error}))))))) :error error})))))))
finish-handler (fn [] finish-handler (fn []
(when finish-handler (when finish-handler
(finish-handler)) (finish-handler)))]
(ui-handler/re-render-file!))]
(-> (p/all (map write-file-f files)) (-> (p/all (map write-file-f files))
(p/then (fn [] (p/then (fn []
(finish-handler) (finish-handler)))
(when chan
(async/put! chan true))))
(p/catch (fn [error] (p/catch (fn [error]
(println "Alter files failed:") (println "Alter files failed:")
(js/console.error error) (js/console.error error))))))
(async/put! chan false))))))
(defn run-writes-chan! (defn alter-files
[] [repo files {:keys [reset? update-db?]
(let [chan (state/get-file-write-chan)] :or {reset? false
(async/go-loop [] update-db? true}
(let [args (async/<! chan)] :as opts}]
;; return a channel ;; old file content
(try (let [file->content (let [paths (map first files)]
(<p! (apply alter-files-handler! args)) (zipmap paths
(catch js/Error e (map (fn [path] (db/get-file repo path)) paths)))]
(log/error :file/write-failed e)))) ;; update db
(recur)) (when update-db?
chan)) (doseq [[path content] files]
(if reset?
(reset-file! repo path content)
(db/set-file-content! repo path content))))
(alter-files-handler! repo files opts file->content)))
(defn watch-for-current-graph-dir! (defn watch-for-current-graph-dir!
[] []

6
src/main/frontend/handler/ui.cljs
View File

@ -99,12 +99,6 @@
(doseq [component (state/get-custom-query-components)] (doseq [component (state/get-custom-query-components)]
(rum/request-render component))))) (rum/request-render component)))))
(defn re-render-file!
[]
(when-let [component (state/get-file-component)]
(when (= :file (state/get-current-route))
(rum/request-render component))))
(defn highlight-element! (defn highlight-element!
[fragment] [fragment]
(let [id (and (let [id (and

21
src/main/frontend/modules/file/core.cljs
View File

@ -1,13 +1,13 @@
(ns frontend.modules.file.core (ns frontend.modules.file.core
(:require [cljs.core.async :as async] (:require [clojure.string :as string]
[clojure.string :as string]
[frontend.config :as config] [frontend.config :as config]
[frontend.date :as date] [frontend.date :as date]
[frontend.db :as db] [frontend.db :as db]
[frontend.db.utils :as db-utils] [frontend.db.utils :as db-utils]
[frontend.state :as state] [frontend.state :as state]
[frontend.util :as util] [frontend.util :as util]
[frontend.util.property :as property])) [frontend.util.property :as property]
[frontend.handler.file :as file-handler]))
(defn- indented-block-content (defn- indented-block-content
[content spaces-tabs] [content spaces-tabs]
@ -101,16 +101,6 @@
(def init-level 1) (def init-level 1)
(defn push-to-write-chan
[files & opts]
(let [repo (state/get-current-repo)
chan (state/get-file-write-chan)]
(assert (some? chan) "File write chan shouldn't be nil")
(let [chan-callback (:chan-callback opts)]
(async/put! chan [repo files opts])
(when chan-callback
(chan-callback)))))
(defn- transact-file-tx-if-not-exists! (defn- transact-file-tx-if-not-exists!
[page ok-handler] [page ok-handler]
(when-let [repo (state/get-current-repo)] (when-let [repo (state/get-current-repo)]
@ -143,8 +133,9 @@
file-path (-> (db-utils/entity file-db-id) :file/path) file-path (-> (db-utils/entity file-db-id) :file/path)
_ (assert (string? file-path) "File path should satisfy string?") _ (assert (string? file-path) "File path should satisfy string?")
;; FIXME: name conflicts between multiple graphs ;; FIXME: name conflicts between multiple graphs
files [[file-path new-content]]] files [[file-path new-content]]
(push-to-write-chan files))) repo (state/get-current-repo)]
(file-handler/alter-files-handler! repo files {} {})))
(defn save-tree (defn save-tree
[page-block tree] [page-block tree]

11
src/main/frontend/modules/outliner/file.cljs
View File

@ -12,7 +12,6 @@
[lambdaisland.glogi :as log] [lambdaisland.glogi :as log]
[frontend.state :as state])) [frontend.state :as state]))
(defonce write-chan (async/chan 100))
(defonce write-chan-batch-buf (atom [])) (defonce write-chan-batch-buf (atom []))
(def batch-write-interval 1000) (def batch-write-interval 1000)
@ -27,8 +26,8 @@
page-db-id (:db/id page-block) page-db-id (:db/id page-block)
blocks-count (model/get-page-blocks-count repo page-db-id)] blocks-count (model/get-page-blocks-count repo page-db-id)]
(if (and (> blocks-count 500) (if (and (> blocks-count 500)
(not (state/input-idle? repo :diff 3000))) ; long page (not (state/input-idle? repo :diff 3000))) ; long page
(async/put! write-chan [repo page-db-id]) (async/put! (state/get-file-write-chan) [repo page-db-id])
(let [blocks (model/get-page-blocks-no-cache repo (:block/name page-block))] (let [blocks (model/get-page-blocks-no-cache repo (:block/name page-block))]
(when-not (and (= 1 (count blocks)) (when-not (and (= 1 (count blocks))
(string/blank? (:block/content (first blocks))) (string/blank? (:block/content (first blocks)))
@ -59,9 +58,11 @@
"Write file failed, can't find the current page!" "Write file failed, can't find the current page!"
:error) :error)
(when-let [repo (state/get-current-repo)] (when-let [repo (state/get-current-repo)]
(async/put! write-chan [repo page-db-id])))) (if (:graph/importing @state/state) ; write immediately
(write-files! [[repo page-db-id]])
(async/put! (state/get-file-write-chan) [repo page-db-id])))))
(util/batch write-chan (util/batch (state/get-file-write-chan)
batch-write-interval batch-write-interval
write-files! write-files!
write-chan-batch-buf) write-chan-batch-buf)

12
src/main/frontend/modules/outliner/pipeline.cljs
View File

@ -70,15 +70,19 @@
tx (util/concat-without-nil truncate-refs-tx refs-tx) tx (util/concat-without-nil truncate-refs-tx refs-tx)
tx-report' (if (seq tx) tx-report' (if (seq tx)
(let [refs-tx-data' (:tx-data (db/transact! repo tx {:outliner/transact? true (let [refs-tx-data' (:tx-data (db/transact! repo tx {:outliner/transact? true
:compute-new-refs? true}))] :compute-new-refs? true}))]
;; merge ;; merge
(assoc tx-report :tx-data (concat (:tx-data tx-report) refs-tx-data'))) (assoc tx-report :tx-data (concat (:tx-data tx-report) refs-tx-data')))
tx-report)] tx-report)
(react/refresh! repo tx-report') importing? (:graph/importing @state/state)]
(when-not importing?
(react/refresh! repo tx-report'))
(doseq [p (seq pages)] (doseq [p (seq pages)]
(updated-page-hook tx-report p)) (updated-page-hook tx-report p))
(when (and state/lsp-enabled? (seq blocks))
(when (and state/lsp-enabled? (seq blocks) (not importing?))
(state/pub-event! [:plugin/hook-db-tx (state/pub-event! [:plugin/hook-db-tx
{:blocks blocks {:blocks blocks
:tx-data (:tx-data tx-report) :tx-data (:tx-data tx-report)

21
src/main/frontend/state.cljs
View File

@ -24,11 +24,11 @@
(atom (atom
{:route-match nil {:route-match nil
:today nil :today nil
:system/events (async/chan 100) :system/events (async/chan 1000)
:db/batch-txs (async/chan 100) :db/batch-txs (async/chan 1000)
:file/writes (async/chan 100) :file/writes (async/chan 10000)
:file/unlinked-dirs #{} :file/unlinked-dirs #{}
:reactive/custom-queries (async/chan 100) :reactive/custom-queries (async/chan 1000)
:notification/show? false :notification/show? false
:notification/content nil :notification/content nil
:repo/loading-files? {} :repo/loading-files? {}
@ -232,7 +232,9 @@
:encryption/graph-parsing? false :encryption/graph-parsing? false
:ui/find-in-page nil :ui/find-in-page nil
:graph/importing nil
:graph/importing-state {}
}))) })))
;; block uuid -> {content(String) -> ast} ;; block uuid -> {content(String) -> ast}
@ -1005,10 +1007,6 @@
[] []
(set-state! :ui/file-component nil)) (set-state! :ui/file-component nil))
(defn get-file-component
[]
(get @state :ui/file-component))
(defn set-journals-length! (defn set-journals-length!
[value] [value]
(when value (when value
@ -1158,11 +1156,6 @@
[] []
(:reactive/custom-queries @state)) (:reactive/custom-queries @state))
(defn get-write-chan-length
[]
(let [c (get-file-write-chan)]
(count (gobj/get c "buf"))))
(defn get-left-sidebar-open? (defn get-left-sidebar-open?
[] []
(get-in @state [:ui/left-sidebar-open?])) (get-in @state [:ui/left-sidebar-open?]))