greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

wip: export refactor 

1. Move fns to worker
2. Don't rely on :block/file, use db blocks to build page content
pull/10981/head
Tienson Qin 2024-01-21 18:15:44 +08:00
parent 760a9f8c25
commit e571c571ec
10 changed files with 254 additions and 431 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/frontend/components/export.cljs
View File

@ -22,10 +22,10 @@
[:h1.title (t :export)]
[:ul.mr-1
[:li.mb-4
[:a.font-medium {:on-click #(export/export-repo-as-edn-v2! current-repo)}
[:a.font-medium {:on-click #(export/export-repo-as-edn! current-repo)}
(t :export-edn)]]
[:li.mb-4
[:a.font-medium {:on-click #(export/export-repo-as-json-v2! current-repo)}
[:a.font-medium {:on-click #(export/export-repo-as-json! current-repo)}
(t :export-json)]]
(when (config/db-based-graph? current-repo)
[:li.mb-4

8
src/main/frontend/components/file.cljs
View File

@ -57,13 +57,7 @@
(if (or (nil? modified-at) (zero? modified-at))
(t :file/no-data)
(date/get-date-time-string
(t/to-default-time-zone (tc/to-date-time modified-at))))]])
(when-not mobile?
[:td [:a.text-sm
{:on-click (fn [_e]
(export-handler/download-file! file))}
[:span (t :download)]]])]))]])))
(t/to-default-time-zone (tc/to-date-time modified-at))))]])]))]])))
(rum/defc files
[]

19
src/main/frontend/db_worker.cljs
View File

@ -15,6 +15,7 @@
[logseq.db.sqlite.util :as sqlite-util]
[frontend.worker.state :as worker-state]
[frontend.worker.file :as file]
[frontend.worker.export :as worker-export]
[logseq.db :as ldb]
[frontend.worker.rtc.op-mem-layer :as op-mem-layer]
[frontend.worker.rtc.db-listener :as rtc-db-listener]
@ -405,6 +406,24 @@
(worker-state/set-new-state! new-state)
nil))
;; Export
(block->content
[this repo block-uuid-or-page-name tree->file-opts context]
(when-let [conn (worker-state/get-datascript-conn repo)]
(worker-export/block->content repo @conn block-uuid-or-page-name
(edn/read-string tree->file-opts)
(edn/read-string context))))
(get-all-pages
[this repo]
(when-let [conn (worker-state/get-datascript-conn repo)]
(bean/->js (worker-export/get-all-pages repo @conn))))
(get-all-page->content
[this repo]
(when-let [conn (worker-state/get-datascript-conn repo)]
(bean/->js (worker-export/get-all-page->content repo @conn))))
;; RTC
(rtc-start
[this repo token]

369
src/main/frontend/handler/export.cljs
View File

@ -5,74 +5,24 @@
[clojure.set :as s]
[clojure.string :as string]
[clojure.walk :as walk]
[datascript.core :as d]
[frontend.config :as config]
[frontend.db :as db]
[frontend.extensions.zip :as zip]
[frontend.external.roam-export :as roam-export]
[frontend.format.mldoc :as mldoc]
[frontend.handler.notification :as notification]
[frontend.mobile.util :as mobile-util]
[frontend.modules.file.core :as outliner-file]
[frontend.modules.outliner.tree :as outliner-tree]
[logseq.publishing.html :as publish-html]
[frontend.state :as state]
[frontend.util :as util]
[frontend.handler.property.file :as property-file]
[goog.dom :as gdom]
[lambdaisland.glogi :as log]
[logseq.graph-parser.property :as gp-property]
[logseq.common.util.block-ref :as block-ref]
[logseq.common.util.page-ref :as page-ref]
[promesa.core :as p]
[frontend.persist-db :as persist-db])
[frontend.persist-db :as persist-db]
[frontend.persist-db.browser :as db-browser]
[cljs-bean.core :as bean])
(:import
[goog.string StringBuffer]))
(defn- get-page-content
[repo page]
(outliner-file/tree->file-content
(outliner-tree/blocks->vec-tree
(db/get-page-blocks-no-cache repo page) page) {:init-level 1}))
(defn- get-file-content
[repo file-path]
(if-let [page-name
(ffirst (d/q '[:find ?pn
:in $ ?path
:where
[?p :block/file ?f]
[?p :block/name ?pn]
[?f :file/path ?path]]
(db/get-db repo) file-path))]
(get-page-content repo page-name)
(ffirst
(d/q '[:find ?content
:in $ ?path
:where
[?f :file/path ?path]
[?f :file/content ?content]]
(db/get-db repo) file-path))))
(defn- get-blocks-contents
[repo root-block-uuid]
(->
(db/get-block-and-children repo root-block-uuid)
(outliner-tree/blocks->vec-tree (str root-block-uuid))
(outliner-file/tree->file-content {:init-level 1})))
(defn download-file!
[file-path]
(when-let [repo (state/get-current-repo)]
(when-let [content (get-file-content repo file-path)]
(let [data (js/Blob. ["\ufeff" (array content)] ; prepend BOM
(clj->js {:type "text/plain;charset=utf-8,"}))
anchor (gdom/getElement "download")
url (js/window.URL.createObjectURL data)]
(.setAttribute anchor "href" url)
(.setAttribute anchor "download" file-path)
(.click anchor)))))
(defn download-repo-as-html!
"download public pages as html"
[repo]
@ -98,28 +48,21 @@
(.setAttribute anchor "download" "index.html")
(.click anchor))))))
(defn- get-file-contents
([repo]
(get-file-contents repo {:init-level 1}))
([repo file-opts]
(let [db (db/get-db repo)]
(->> (d/q '[:find ?n ?fp
:where
[?e :block/file ?f]
[?f :file/path ?fp]
[?e :block/name ?n]] db)
(mapv (fn [[page-name file-path]]
[file-path
(outliner-file/tree->file-content
(outliner-tree/blocks->vec-tree
(db/get-page-blocks-no-cache page-name) page-name)
file-opts)]))))))
(defn- <get-all-page->content
[repo]
(when-let [^object worker @db-browser/*worker]
(.get-all-page->content worker repo)))
(defn export-repo-as-zip!
[repo]
(let [files (get-file-contents repo)
[owner repo-name] (util/get-git-owner-and-repo repo)
repo-name (str owner "-" repo-name)]
(p/let [page->content (<get-all-page->content repo)
[owner repo-name] (util/get-git-owner-and-repo repo)
repo-name (str owner "-" repo-name)
files (map (fn [[title content]]
[(str title "." (if (= :org (state/get-preferred-format))
"org"
"md"))
content]) page->content)]
(when (seq files)
(p/let [zipfile (zip/make-zip repo-name files repo)]
(when-let [anchor (gdom/getElement "download")]
@ -127,137 +70,6 @@
(.setAttribute anchor "download" (.-name zipfile))
(.click anchor))))))
(defn- get-embed-pages-from-ast [ast]
(let [result (transient #{})]
(doseq [item ast]
(walk/prewalk (fn [i]
(cond
(and (vector? i)
(= "Macro" (first i))
(= "embed" (some-> (:name (second i))
(string/lower-case)))
(some-> (:arguments (second i))
first
page-ref/page-ref?))
(let [arguments (:arguments (second i))
page-ref (first arguments)
page-name (-> page-ref
(subs 2)
(#(subs % 0 (- (count %) 2)))
(string/lower-case))]
(conj! result page-name)
i)
:else
i))
item))
(persistent! result)))
(defn- get-embed-blocks-from-ast [ast]
(let [result (transient #{})]
(doseq [item ast]
(walk/prewalk (fn [i]
(cond
(and (vector? i)
(= "Macro" (first i))
(= "embed" (some-> (:name (second i))
(string/lower-case)))
(some-> (:arguments (second i))
(first)
block-ref/string-block-ref?))
(let [arguments (:arguments (second i))
block-uuid (block-ref/get-string-block-ref-id (first arguments))]
(conj! result block-uuid)
i)
:else
i)) item))
(persistent! result)))
(defn- get-block-refs-from-ast [ast]
(let [result (transient #{})]
(doseq [item ast]
(walk/prewalk (fn [i]
(cond
(and (vector? i)
(= "Block_ref" (first i))
(some? (second i)))
(let [block-uuid (second i)]
(conj! result block-uuid)
i)
:else
i)) item))
(persistent! result)))
(declare get-page-page&block-refs)
(defn get-block-page&block-refs [repo block-uuid embed-pages embed-blocks block-refs]
(let [block (db/entity [:block/uuid (uuid block-uuid)])
block-content (get-blocks-contents repo (:block/uuid block))
format (:block/format block)
ast (mldoc/->edn block-content format)
embed-pages-new (get-embed-pages-from-ast ast)
embed-blocks-new (get-embed-blocks-from-ast ast)
block-refs-new (get-block-refs-from-ast ast)
embed-pages-diff (s/difference embed-pages-new embed-pages)
embed-blocks-diff (s/difference embed-blocks-new embed-blocks)
block-refs-diff (s/difference block-refs-new block-refs)
embed-pages* (s/union embed-pages-new embed-pages)
embed-blocks* (s/union embed-blocks-new embed-blocks)
block-refs* (s/union block-refs-new block-refs)
[embed-pages-1 embed-blocks-1 block-refs-1]
(->>
(mapv (fn [page-name]
(let [{:keys [embed-pages embed-blocks block-refs]}
(get-page-page&block-refs repo page-name embed-pages* embed-blocks* block-refs*)]
[embed-pages embed-blocks block-refs])) embed-pages-diff)
(apply mapv vector) ; [[1 2 3] [4 5 6] [7 8 9]] -> [[1 4 7] [2 5 8] [3 6 9]]
(mapv #(apply s/union %)))
[embed-pages-2 embed-blocks-2 block-refs-2]
(->>
(mapv (fn [block-uuid]
(let [{:keys [embed-pages embed-blocks block-refs]}
(get-block-page&block-refs repo block-uuid embed-pages* embed-blocks* block-refs*)]
[embed-pages embed-blocks block-refs])) (s/union embed-blocks-diff block-refs-diff))
(apply mapv vector)
(mapv #(apply s/union %)))]
{:embed-pages (s/union embed-pages-1 embed-pages-2 embed-pages*)
:embed-blocks (s/union embed-blocks-1 embed-blocks-2 embed-blocks*)
:block-refs (s/union block-refs-1 block-refs-2 block-refs*)}))
(defn get-page-page&block-refs [repo page-name embed-pages embed-blocks block-refs]
(let [page-name* (util/page-name-sanity-lc page-name)
page-content (get-page-content repo page-name*)
format (:block/format (db/entity [:block/name page-name*]))
ast (mldoc/->edn page-content format)
embed-pages-new (get-embed-pages-from-ast ast)
embed-blocks-new (get-embed-blocks-from-ast ast)
block-refs-new (get-block-refs-from-ast ast)
embed-pages-diff (s/difference embed-pages-new embed-pages)
embed-blocks-diff (s/difference embed-blocks-new embed-blocks)
block-refs-diff (s/difference block-refs-new block-refs)
embed-pages* (s/union embed-pages-new embed-pages)
embed-blocks* (s/union embed-blocks-new embed-blocks)
block-refs* (s/union block-refs-new block-refs)
[embed-pages-1 embed-blocks-1 block-refs-1]
(->>
(mapv (fn [page-name]
(let [{:keys [embed-pages embed-blocks block-refs]}
(get-page-page&block-refs repo page-name embed-pages* embed-blocks* block-refs*)]
[embed-pages embed-blocks block-refs])) embed-pages-diff)
(apply mapv vector)
(mapv #(apply s/union %)))
[embed-pages-2 embed-blocks-2 block-refs-2]
(->>
(mapv (fn [block-uuid]
(let [{:keys [embed-pages embed-blocks block-refs]}
(get-block-page&block-refs repo block-uuid embed-pages* embed-blocks* block-refs*)]
[embed-pages embed-blocks block-refs])) (s/union embed-blocks-diff block-refs-diff))
(apply mapv vector)
(mapv #(apply s/union %)))]
{:embed-pages (s/union embed-pages-1 embed-pages-2 embed-pages*)
:embed-blocks (s/union embed-blocks-1 embed-blocks-2 embed-blocks*)
:block-refs (s/union block-refs-1 block-refs-2 block-refs*)}))
(defn- export-file-on-mobile [data path]
(p/catch
(.writeFile Filesystem (clj->js {:path path
@ -294,56 +106,26 @@
x))
vec-tree))
(defn- safe-keywordize
[block]
(update block :block/properties
(fn [properties]
(when (seq properties)
(->> (filter (fn [[k _v]]
(gp-property/valid-property-name? (str k))) properties)
(into {}))))))
(defn- <get-all-pages
[repo]
(when-let [^object worker @db-browser/*worker]
(.get-all-pages worker repo)))
(defn- blocks
[repo db]
{:version 1
:blocks
(->> (d/q '[:find (pull ?b [*])
:in $
:where
[?b :block/original-name]
[?b :block/name]] db)
(map (fn [[{:block/keys [name] :as page}]]
(let [whiteboard? (contains? (set (:block/type page)) "whiteboard")
blocks (db/get-page-blocks-no-cache
(state/get-current-repo)
name
{:transform? false})
blocks' (if whiteboard?
blocks
(map (fn [b]
(let [b' (if (seq (:block/properties b))
(update b :block/content
(fn [content]
(property-file/remove-properties-when-file-based
repo (:block/format b) content)))
b)]
(safe-keywordize b'))) blocks))
children (if whiteboard?
blocks'
(outliner-tree/blocks->vec-tree blocks' name))
page' (safe-keywordize page)]
(assoc page' :block/children children))))
(nested-select-keys
[:block/id
:block/type
:block/page-name
:block/properties
:block/format
:block/children
:block/content
:block/created-at
:block/updated-at]))})
(defn- <build-blocks
[repo]
(p/let [pages (<get-all-pages repo)]
{:version 1
:blocks
(nested-select-keys pages
[:block/id
:block/type
:block/page-name
:block/properties
:block/format
:block/children
:block/content
:block/created-at
:block/updated-at])}))
(defn- file-name [repo extension]
(-> (string/replace repo config/local-db-prefix "")
@ -351,15 +133,15 @@
(str "_" (quot (util/time-ms) 1000))
(str "." (string/lower-case (name extension)))))
(defn- export-repo-as-edn-str [repo]
(when-let [db (db/get-db repo)]
(defn- <export-repo-as-edn-str [repo]
(p/let [result (<build-blocks repo)]
(let [sb (StringBuffer.)]
(pprint/pprint (blocks repo db) (StringBufferWriter. sb))
(pprint/pprint result (StringBufferWriter. sb))
(str sb))))
(defn export-repo-as-edn-v2!
(defn export-repo-as-edn!
[repo]
(when-let [edn-str (export-repo-as-edn-str repo)]
(when-let [edn-str (<export-repo-as-edn-str repo)]
(let [data-str (some->> edn-str
js/encodeURIComponent
(str "data:text/edn;charset=utf-8,"))
@ -380,23 +162,22 @@
x))
vec-tree))
(defn export-repo-as-json-v2!
(defn export-repo-as-json!
[repo]
(when-let [db (db/get-db repo)]
(let [json-str
(-> (blocks repo db)
nested-update-id
clj->js
js/JSON.stringify)
(p/let [result (<build-blocks repo)
json-str (-> result
nested-update-id
clj->js
js/JSON.stringify)
filename (file-name repo :json)
data-str (str "data:text/json;charset=utf-8,"
(js/encodeURIComponent json-str))]
(if (mobile-util/native-platform?)
(export-file-on-mobile json-str filename)
(when-let [anchor (gdom/getElement "download-as-json-v2")]
(.setAttribute anchor "href" data-str)
(.setAttribute anchor "download" filename)
(.click anchor))))))
(if (mobile-util/native-platform?)
(export-file-on-mobile json-str filename)
(when-let [anchor (gdom/getElement "download-as-json-v2")]
(.setAttribute anchor "href" data-str)
(.setAttribute anchor "download" filename)
(.click anchor)))))
(defn export-repo-as-sqlite-db!
[repo]
@ -415,39 +196,25 @@
;; https://roamresearch.com/#/app/help/page/Nxz8u0vXU
;; export to roam json according to above spec
(defn- roam-json [db]
(->> (d/q '[:find (pull ?b [*])
:in $
:where
[?b :block/file]
[?b :block/original-name]
[?b :block/name]] db)
(map (fn [[{:block/keys [name] :as page}]]
(assoc page
:block/children
(outliner-tree/blocks->vec-tree
(db/get-page-blocks-no-cache
(state/get-current-repo)
name
{:transform? false}) name))))
(roam-export/traverse
[:page/title
:block/string
:block/uid
:block/children])))
(defn- <roam-data [repo]
(p/let [pages (<get-all-pages repo)]
(let [non-empty-pages (remove #(empty? (:block/children %)) pages)]
(roam-export/traverse
[:page/title
:block/string
:block/uid
:block/children]
non-empty-pages))))
(defn export-repo-as-roam-json!
[repo]
(when-let [db (db/get-db repo)]
(let [json-str
(-> (roam-json db)
clj->js
js/JSON.stringify)
(p/let [data (<roam-data repo)
json-str (-> data
bean/->js
js/JSON.stringify)
data-str (str "data:text/json;charset=utf-8,"
(js/encodeURIComponent json-str))]
(when-let [anchor (gdom/getElement "download-as-roam-json")]
(.setAttribute anchor "href" data-str)
(.setAttribute anchor "download" (file-name (str repo "_roam") :json))
(.click anchor)))))
(when-let [anchor (gdom/getElement "download-as-roam-json")]
(.setAttribute anchor "href" data-str)
(.setAttribute anchor "download" (file-name (str repo "_roam") :json))
(.click anchor))))

50
src/main/frontend/handler/export/common.cljs
View File

@ -5,7 +5,6 @@
(:refer-clojure :exclude [map filter mapcat concat remove])
(:require [cljs.core.match :refer [match]]
[clojure.string :as string]
[datascript.core :as d]
[frontend.db :as db]
[frontend.format.mldoc :as mldoc]
[frontend.modules.file.core :as outliner-file]
@ -15,7 +14,9 @@
[logseq.common.util :as common-util]
[frontend.handler.property.util :as pu]
[malli.core :as m]
[malli.util :as mu]))
[malli.util :as mu]
[frontend.db.async :as db-async]
[promesa.core :as p]))
;;; TODO: split frontend.handler.export.text related states
(def ^:dynamic *state*
@ -179,38 +180,33 @@
ast-content)))
inline-coll)))
(defn- get-file-contents
(defn- <get-file-contents
[repo]
(let [db (db/get-db repo)]
(->> (d/q '[:find ?fp
:where
[?e :block/file ?f]
[?f :file/path ?fp]] db)
(mapv (fn [[file-path]]
[file-path
(db/get-file file-path)])))))
(db-async/<q repo '[:find ?pn ?fp ?fc
:where
[?e :block/original-name ?pn]
[?e :block/file ?f]
[?f :file/path ?fp]
[?f :file/content ?fc]]))
(defn- get-md-file-contents
(defn- <get-md-file-contents
[repo]
(filterv (fn [[path _]]
(let [path (string/lower-case path)]
(re-find #"\.(?:md|markdown)$" path)))
(get-file-contents repo)))
(p/let [result (<get-file-contents repo)]
(filterv (fn [[path _]]
(let [path (string/lower-case path)]
(re-find #"\.(?:md|markdown)$" path)))
result)))
(defn get-file-contents-with-suffix
(defn <get-file-contents-with-suffix
[repo]
(let [db (db/get-db repo)
md-files (get-md-file-contents repo)]
(p/let [md-files (<get-md-file-contents repo)]
(->>
md-files
(mapv (fn [[path content]] {:path path :content content
:names (d/q '[:find [?n ?n2]
:in $ ?p
:where [?e :file/path ?p]
[?e2 :block/file ?e]
[?e2 :block/name ?n]
[?e2 :block/original-name ?n2]] db path)
:format (common-util/get-format path)})))))
(mapv (fn [[page-title path content]]
{:path path
:content content
:title page-title
:format (common-util/get-format path)})))))
;;; utils (ends)

23
src/main/frontend/handler/export/opml.cljs
View File

@ -450,21 +450,22 @@
"options see also `export-blocks-as-opml`"
[files options]
(mapv
(fn [{:keys [path content names format]}]
(when (first names)
(fn [{:keys [path content title format]}]
(when title
(util/profile (print-str :export-files-as-opml path)
[path (export-helper content format options :title (first names))])))
[path (export-helper content format options :title title)])))
files))
(defn export-repo-as-opml!
[repo]
(when-let [files (common/get-file-contents-with-suffix repo)]
(let [files (export-files-as-opml files nil)
zip-file-name (str repo "_opml_" (quot (util/time-ms) 1000))]
(p/let [zipfile (zip/make-zip zip-file-name files repo)]
(when-let [anchor (gdom/getElement "export-as-opml")]
(.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
(.setAttribute anchor "download" (.-name zipfile))
(.click anchor))))))
(p/let [files (common/<get-file-contents-with-suffix repo)]
(when (seq files)
(let [files (export-files-as-opml files nil)
zip-file-name (str repo "_opml_" (quot (util/time-ms) 1000))]
(p/let [zipfile (zip/make-zip zip-file-name files repo)]
(when-let [anchor (gdom/getElement "export-as-opml")]
(.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
(.setAttribute anchor "download" (.-name zipfile))
(.click anchor)))))))
;;; export fns (ends)

21
src/main/frontend/handler/export/text.cljs
View File

@ -495,8 +495,8 @@
"options see also `export-blocks-as-markdown`"
[files options]
(mapv
(fn [{:keys [path content names format]}]
(when (first names)
(fn [{:keys [path content title format]}]
(when title
(util/profile (print-str :export-files-as-markdown path)
[path (export-helper content format options)])))
files))
@ -504,13 +504,14 @@
(defn export-repo-as-markdown!
"TODO: indent-style and remove-options"
[repo]
(when-let [files (util/profile :get-file-content (common/get-file-contents-with-suffix repo))]
(let [files (export-files-as-markdown files nil)
zip-file-name (str repo "_markdown_" (quot (util/time-ms) 1000))]
(p/let [zipfile (zip/make-zip zip-file-name files repo)]
(when-let [anchor (gdom/getElement "export-as-markdown")]
(.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
(.setAttribute anchor "download" (.-name zipfile))
(.click anchor))))))
(p/let [files (util/profile :get-file-content (common/<get-file-contents-with-suffix repo))]
(when (seq files)
(let [files (export-files-as-markdown files nil)
zip-file-name (str repo "_markdown_" (quot (util/time-ms) 1000))]
(p/let [zipfile (zip/make-zip zip-file-name files repo)]
(when-let [anchor (gdom/getElement "export-as-markdown")]
(.setAttribute anchor "href" (js/window.URL.createObjectURL zipfile))
(.setAttribute anchor "download" (.-name zipfile))
(.click anchor)))))))
;;; export fns (ends)

28
src/main/frontend/handler/whiteboard.cljs
View File

@ -18,8 +18,7 @@
[clojure.set :as set]
[clojure.string :as string]
[cljs-bean.core :as bean]
[logseq.db.sqlite.util :as sqlite-util]
[clojure.data :as data]))
[logseq.db.sqlite.util :as sqlite-util]))
(defn js->clj-keywordize
[obj]
@ -255,39 +254,18 @@
blocks (:block/_page react-page)]
(whiteboard-clj->tldr react-page blocks)))
(defn- get-whiteboard-blocks
"Given a page, return all the logseq blocks (exclude all shapes)"
[page-name]
(let [blocks (model/get-page-blocks-no-cache page-name)]
(remove pu/shape-block? blocks)))
(defn- get-last-root-block
"Get the last root Logseq block in the page. Main purpose is to calculate the new :block/left id"
[page-name]
(let [page-id (:db/id (model/get-page page-name))
blocks (get-whiteboard-blocks page-name)
root-blocks (filter (fn [block] (= page-id (:db/id (:block/parent block)))) blocks)
root-block-left-ids (->> root-blocks
(map (fn [block] (get-in block [:block/left :db/id] nil)))
(remove nil?)
(set))
blocks-with-no-next (remove #(root-block-left-ids (:db/id %)) root-blocks)]
(when (seq blocks-with-no-next) (first blocks-with-no-next))))
(defn <add-new-block!
[page-name content]
(p/let [repo (state/get-current-repo)
new-block-id (db/new-block-id)
page-entity (model/get-page page-name)
last-root-block (or (get-last-root-block page-name) page-entity)
tx (sqlite-util/block-with-timestamps
{:block/left (select-keys last-root-block [:db/id])
:block/uuid new-block-id
{:block/uuid new-block-id
:block/content (or content "")
:block/format :markdown
:block/page {:block/name (util/page-name-sanity-lc page-name)
:block/original-name page-name}
:block/parent {:block/name page-name}})
:block/parent (:db/id page-entity)})
_ (db/transact! repo [tx] {:whiteboard/transact? true})]
new-block-id))

68
src/main/frontend/worker/export.cljs Normal file
View File

@ -0,0 +1,68 @@
(ns frontend.worker.export
"Export data"
(:require [logseq.db :as ldb]
[logseq.outliner.tree :as otree]
[frontend.worker.file.core :as worker-file]
[datascript.core :as d]
[logseq.common.util :as common-util]
[logseq.graph-parser.property :as gp-property]
[datascript.core :as d]))
(defn block->content
"Converts a block including its children (recursively) to plain-text."
[repo db root-block-uuid-or-page-name tree->file-opts context]
(let [root-block-uuid (or
(and (uuid? root-block-uuid-or-page-name) root-block-uuid-or-page-name)
(:block/uuid (d/entity db [:block/name (common-util/page-name-sanity-lc
root-block-uuid-or-page-name)])))
init-level (or (:init-level tree->file-opts)
(if (uuid? root-block-uuid-or-page-name) 1 0))
blocks (ldb/get-block-and-children repo db root-block-uuid)
tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))]
(worker-file/tree->file-content repo db tree
(assoc tree->file-opts :init-level init-level)
context)))
(defn- safe-keywordize
[block]
(update block :block/properties
(fn [properties]
(when (seq properties)
(->> (filter (fn [[k _v]]
(gp-property/valid-property-name? (str k))) properties)
(into {}))))))
(defn get-all-pages
"Get all pages and their children blocks."
[repo db]
(->> (d/q '[:find (pull ?b [*])
:in $
:where
[?b :block/original-name]
[?b :block/name]] db)
(map (fn [[{:block/keys [name] :as page}]]
(let [whiteboard? (contains? (set (:block/type page)) "whiteboard")
blocks (ldb/get-page-blocks db name {})
blocks' (if whiteboard?
blocks
(map (fn [b]
(let [b' (if (seq (:block/properties b))
(update b :block/content
(fn [content]
(gp-property/remove-properties (:block/format b) content)))
b)]
(safe-keywordize b'))) blocks))
children (if whiteboard?
blocks'
(otree/blocks->vec-tree repo db blocks' name))
page' (safe-keywordize page)]
(assoc page' :block/children children))))))
(defn get-all-page->content
[repo db]
(->> (d/datoms db :avet :block/name)
(map (fn [d]
(let [e (d/entity db (:e d))]
[(:block/original-name e)
(block->content repo db (:v d) {} {})])))))

95
src/test/frontend/handler/export_test.cljs
View File

@ -15,11 +15,11 @@
(string/trim "
- 1
id:: 61506710-484c-46d5-9983-3d1651ec02c8
- 2
id:: 61506711-5638-4899-ad78-187bdc2eaffc
- 3
id:: 61506712-3007-407e-b6d3-d008a8dfa88b
- ((61506712-3007-407e-b6d3-d008a8dfa88b))
- 2
id:: 61506711-5638-4899-ad78-187bdc2eaffc
- 3
id:: 61506712-3007-407e-b6d3-d008a8dfa88b
- ((61506712-3007-407e-b6d3-d008a8dfa88b))
- 4
id:: 61506712-b8a7-491d-ad84-b71651c3fdab")}
{:file/path "pages/page2.md"
@ -27,7 +27,7 @@
(string/trim "
- 3
id:: 97a00e55-48c3-48d8-b9ca-417b16e3a616
- {{embed [[page1]]}}")}])
- {{embed [[page1]]}}")}])
(use-fixtures :once
{:before (fn []
@ -44,18 +44,18 @@
{:remove-options #{:property}})))
(string/trim "
- 1
- 2
- 3
- 3")
- 2
- 3
- 3")
"61506710-484c-46d5-9983-3d1651ec02c8"
(string/trim "
- 3
- 1
- 2
- 3
- 3
- 4")
- 1
- 2
- 3
- 3
- 4")
"97a00e55-48c3-48d8-b9ca-417b16e3a616"))
@ -65,25 +65,25 @@
(string/trim "
- 1
id:: 61506710-484c-46d5-9983-3d1651ec02c8
- 2
id:: 61506711-5638-4899-ad78-187bdc2eaffc
- 3
id:: 61506712-3007-407e-b6d3-d008a8dfa88b
- 3")
- 2
id:: 61506711-5638-4899-ad78-187bdc2eaffc
- 3
id:: 61506712-3007-407e-b6d3-d008a8dfa88b
- 3")
"61506710-484c-46d5-9983-3d1651ec02c8"
(string/trim "
- 3
id:: 97a00e55-48c3-48d8-b9ca-417b16e3a616
- 1
id:: 61506710-484c-46d5-9983-3d1651ec02c8
- 2
id:: 61506711-5638-4899-ad78-187bdc2eaffc
- 3
id:: 61506712-3007-407e-b6d3-d008a8dfa88b
- 3
- 4
id:: 61506712-b8a7-491d-ad84-b71651c3fdab")
- 1
id:: 61506710-484c-46d5-9983-3d1651ec02c8
- 2
id:: 61506711-5638-4899-ad78-187bdc2eaffc
- 3
id:: 61506712-3007-407e-b6d3-d008a8dfa88b
- 3
- 4
id:: 61506712-b8a7-491d-ad84-b71651c3fdab")
"97a00e55-48c3-48d8-b9ca-417b16e3a616"))
(deftest export-blocks-as-markdown-level<N
@ -93,13 +93,13 @@
:other-options {:keep-only-level<=N 2}})))
(string/trim "
- 1
- 2")
- 2")
"61506710-484c-46d5-9983-3d1651ec02c8"
(string/trim "
- 3
- 1
- 4")
- 1
- 4")
"97a00e55-48c3-48d8-b9ca-417b16e3a616"))
(deftest export-blocks-as-markdown-newline-after-block
@ -110,24 +110,24 @@
(string/trim "
- 1
- 2
- 2
- 3
- 3
- 3")
- 3")
"61506710-484c-46d5-9983-3d1651ec02c8"
(string/trim "
- 3
- 1
- 1
- 2
- 2
- 3
- 3
- 3
- 3
- 4")
- 4")
"97a00e55-48c3-48d8-b9ca-417b16e3a616"))
@ -143,12 +143,11 @@
[{:path "pages/page2.md" :content (:file/content (nth test-files 1)) :names ["page2"] :format :markdown}])))
(deftest-async export-repo-as-edn-str
(p/do!
(let [edn-output (edn/read-string
(@#'export/export-repo-as-edn-str (state/get-current-repo)))]
(is (= #{:version :blocks} (set (keys edn-output)))
"Correct top-level keys")
(is (= (sort (concat (map :block/original-name default-db/built-in-pages)
["page1" "page2"]))
(sort (map :block/page-name (:blocks edn-output))))
"Correct pages"))))
(p/let [edn-output (edn/read-string
(@#'export/<export-repo-as-edn-str (state/get-current-repo)))]
(is (= #{:version :blocks} (set (keys edn-output)))
"Correct top-level keys")
(is (= (sort (concat (map :block/original-name default-db/built-in-pages)
["page1" "page2"]))
(sort (map :block/page-name (:blocks edn-output))))
"Correct pages")))