mirror of https://github.com/logseq/logseq
Roam data import prototype
parent
c2df042a96
commit
9fc3f0f21d
|
@ -0,0 +1,30 @@
|
|||
(ns frontend.components.external
|
||||
(:require [rum.core :as rum]
|
||||
[goog.object :as gobj]
|
||||
[clojure.string :as string]
|
||||
[frontend.handler.notification :as notification]
|
||||
[frontend.handler.external :as external-handler]))
|
||||
|
||||
(rum/defc import-cp < rum/reactive
|
||||
[]
|
||||
[:div#import
|
||||
[:h1.title "Import JSON from Roam Research"]
|
||||
[:input
|
||||
{:id "import-roam"
|
||||
:type "file"
|
||||
:on-change (fn [e]
|
||||
(let [file (first (array-seq (.-files (.-target e))))
|
||||
file-name (gobj/get file "name")]
|
||||
(if (string/ends-with? file-name ".json")
|
||||
(let [reader (js/FileReader.)]
|
||||
(set! (.-onload reader)
|
||||
(fn [e]
|
||||
(let [text (.. e -target -result)]
|
||||
(external-handler/import-from-roam-json! text))))
|
||||
(.readAsText reader file))
|
||||
(notification/show! "Please choose a JSON file."
|
||||
:error))))
|
||||
}]
|
||||
|
||||
;; TODO: import status process
|
||||
])
|
|
@ -273,6 +273,9 @@
|
|||
(when current-repo
|
||||
{:title "Settings"
|
||||
:options {:href "/settings"}})
|
||||
(when current-repo
|
||||
{:title "Import"
|
||||
:options {:href "/import"}})
|
||||
{:title [:div.flex-row.flex.justify-between.items-center
|
||||
[:span "Join the community"]
|
||||
svg/discord]
|
||||
|
|
|
@ -0,0 +1,17 @@
|
|||
(ns frontend.external
|
||||
(:require [frontend.external.roam :refer [->Roam]]
|
||||
[frontend.external.protocol :as protocol]))
|
||||
|
||||
(defonce roam-record (->Roam))
|
||||
|
||||
(defn get-record
|
||||
[type]
|
||||
(case type
|
||||
:roam
|
||||
roam-record
|
||||
nil))
|
||||
|
||||
(defn to-markdown-files
|
||||
[type content config]
|
||||
(when-let [record (get-record (keyword type))]
|
||||
(protocol/toMarkdownFiles record content config)))
|
|
@ -2,38 +2,115 @@
|
|||
(:require [frontend.external.protocol :as protocol]
|
||||
[cljs-bean.core :as bean]
|
||||
[medley.core :as medley]
|
||||
[clojure.walk :as walk]))
|
||||
[clojure.walk :as walk]
|
||||
[clojure.string :as string]
|
||||
[frontend.util :as util]))
|
||||
|
||||
(defonce all-refed-uids (atom #{}))
|
||||
(defonce uid->uuid (atom {}))
|
||||
|
||||
(defn- reset-state!
|
||||
[]
|
||||
(reset! all-refed-uids #{})
|
||||
(reset! uid->uuid {}))
|
||||
|
||||
;; DONE: 1. uid converted to a uuid
|
||||
;; DONE: 2. merge pages with same names (case-sensitive)
|
||||
;; DONE: 3. mldoc add support to roam research macros, or we can transform here.
|
||||
;; TODO: 4. mldoc add support to nested links
|
||||
;; TODO: hiccup
|
||||
|
||||
(defonce uid-pattern #"\(\(([a-zA-Z0-9_\\-]{9})\)\)")
|
||||
(defonce macro-pattern #"\{\{([^{}]+)\}\}")
|
||||
|
||||
(defn uid-transform
|
||||
[text]
|
||||
(string/replace text uid-pattern (fn [[_ uid]]
|
||||
(let [id (get @uid->uuid uid uid)]
|
||||
(str "((" id "))")))))
|
||||
|
||||
(defn macro-transform
|
||||
[text]
|
||||
(string/replace text macro-pattern (fn [[original text]]
|
||||
(let [[name arg] (string/split text #": ")]
|
||||
(if name
|
||||
(let [name (case name
|
||||
"[[embed]]" "embed"
|
||||
name)]
|
||||
(util/format "{{{%s %s}}}" name arg))
|
||||
original)))))
|
||||
|
||||
(defn load-all-refed-uids!
|
||||
[data]
|
||||
(let [full-text (atom "")]
|
||||
(walk/postwalk
|
||||
(fn [f]
|
||||
(when (and (map? f) (:string f))
|
||||
(swap! full-text (fn [v] (str v (:string f)))))
|
||||
f)
|
||||
data)
|
||||
(let [uids (->> (re-seq uid-pattern @full-text)
|
||||
(map last)
|
||||
(distinct)
|
||||
(set))]
|
||||
(reset! all-refed-uids uids))))
|
||||
|
||||
(defn transform
|
||||
[text]
|
||||
(-> text
|
||||
(string/replace "{{[[TODO]]}}" "TODO")
|
||||
(string/replace "{{[[DONE]]}}" "DONE")
|
||||
(uid-transform)
|
||||
(macro-transform)))
|
||||
|
||||
;; #"(([a-zA-Z0-9_\\-]{9}))"
|
||||
(declare children->text)
|
||||
(defn child->text
|
||||
[{:keys [uid string children] :as child}]
|
||||
(when-not (get @uid->uuid uid)
|
||||
[{:keys [uid string children] :as child} level]
|
||||
(when-not (and (get @uid->uuid uid) uid)
|
||||
(swap! uid->uuid assoc uid (medley/random-uuid)))
|
||||
(let [children-text (->> (map children->text children)
|
||||
(interpose "\n")
|
||||
(apply str))]
|
||||
(let [children-text (children->text children (inc level))
|
||||
level-pattern (apply str (repeat level "#"))
|
||||
properties (when (contains? @all-refed-uids uid)
|
||||
(str
|
||||
(util/format ":PROPERTIES:\n:CUSTOM_ID:%s\n:END:"
|
||||
(str (get @uid->uuid uid)))
|
||||
"\n"))]
|
||||
(if string
|
||||
(str string "\n" children-text)
|
||||
(str level-pattern " " (string/triml string) "\n" properties children-text)
|
||||
children-text)))
|
||||
|
||||
(defn children->text
|
||||
[children]
|
||||
(map child->text children))
|
||||
[children level]
|
||||
(->> (map #(child->text % level) children)
|
||||
(interpose "\n")
|
||||
(apply str)))
|
||||
|
||||
(defn ->file
|
||||
[page-data]
|
||||
(let [{:keys [create-time title children edit-time]} page-data]
|
||||
(let [{:keys [create-time title children edit-time]} page-data
|
||||
initial-level 2]
|
||||
{:title title
|
||||
:created-at create-time
|
||||
:last-modified-at edit-time
|
||||
:text (children->text children)}))
|
||||
:text (when-let [text (children->text children initial-level)]
|
||||
(let [front-matter (util/format "---\ntitle: %s\n---\n\n" title)]
|
||||
(str front-matter (transform text))))}))
|
||||
|
||||
(defn ->files
|
||||
[edn-data]
|
||||
(let [pages-with-data (filter :children edn-data)]
|
||||
(map ->file pages-with-data)))
|
||||
(load-all-refed-uids! edn-data)
|
||||
(let [pages-with-data (filter :children edn-data)
|
||||
files (map ->file edn-data)
|
||||
files (group-by (fn [f] (string/lower-case (:title f)))
|
||||
files)]
|
||||
(map
|
||||
(fn [[_ [fst & others]]]
|
||||
(assoc fst :text
|
||||
(->> (map :text (cons fst others))
|
||||
(interpose "\n")
|
||||
(apply str))))
|
||||
files)))
|
||||
|
||||
(defrecord Roam []
|
||||
protocol/External
|
||||
|
@ -41,7 +118,6 @@
|
|||
(let [data (bean/->clj (js/JSON.parse content))]
|
||||
(->files data))))
|
||||
|
||||
;; (:create-email :create-time :title :children :edit-time :edit-email)
|
||||
(defonce test-roam-json (frontend.db/get-file "same.json"))
|
||||
|
||||
(defonce edn-data (bean/->clj (js/JSON.parse test-roam-json)))
|
||||
(comment
|
||||
(defonce test-roam-json (frontend.db/get-file "same.json"))
|
||||
(defonce edn-data (bean/->clj (js/JSON.parse test-roam-json))))
|
||||
|
|
|
@ -0,0 +1,21 @@
|
|||
(ns frontend.handler.external
|
||||
(:require [frontend.external :as external]
|
||||
[frontend.handler.file :as file-handler]
|
||||
[frontend.handler.notification :as notification]
|
||||
[frontend.state :as state]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(defn import-from-roam-json!
|
||||
[data]
|
||||
(when-let [repo (state/get-current-repo)]
|
||||
(let [files (external/to-markdown-files :roam data {})]
|
||||
(doseq [file files]
|
||||
(try
|
||||
(when-let [text (:text file)]
|
||||
(let [path (str "pages/" (string/replace (:title file) "/" "-") ".md")]
|
||||
(file-handler/alter-file repo path text {})))
|
||||
(catch js/Error e
|
||||
(let [message (str "File " (:title file) " imported failed.")]
|
||||
(println message)
|
||||
(js/console.error e)
|
||||
(notification/show! message :error))))))))
|
|
@ -427,7 +427,8 @@
|
|||
(periodically-pull repo-url pull-now?)
|
||||
(when (and
|
||||
(or (not config/dev?)
|
||||
(= repo-url "https://github.com/tiensonqin/empty-repo"))
|
||||
;; (= repo-url "https://github.com/tiensonqin/empty-repo")
|
||||
)
|
||||
(not (false? (:git-auto-push (state/get-config repo-url)))))
|
||||
(periodically-push-tasks repo-url)))
|
||||
|
||||
|
|
|
@ -52,6 +52,8 @@
|
|||
"Draw"
|
||||
:settings
|
||||
"Settings"
|
||||
:import
|
||||
"Import data into Logseq"
|
||||
"Logseq"))
|
||||
|
||||
(defn set-route-match!
|
||||
|
|
|
@ -6,7 +6,8 @@
|
|||
[frontend.components.page :as page]
|
||||
[frontend.components.diff :as diff]
|
||||
[frontend.components.draw :as draw]
|
||||
[frontend.components.settings :as settings]))
|
||||
[frontend.components.settings :as settings]
|
||||
[frontend.components.external :as external]))
|
||||
|
||||
(def routes
|
||||
[["/"
|
||||
|
@ -55,4 +56,8 @@
|
|||
|
||||
["/settings"
|
||||
{:name :settings
|
||||
:view settings/settings}]])
|
||||
:view settings/settings}]
|
||||
|
||||
["/import"
|
||||
{:name :import
|
||||
:view external/import-cp}]])
|
||||
|
|
Loading…
Reference in New Issue