Roam data import prototype

web/src/main/frontend/components/external.cljs

@@ -0,0 +1,30 @@
+(ns frontend.components.external
+  (:require [rum.core :as rum]
+            [goog.object :as gobj]
+            [clojure.string :as string]
+            [frontend.handler.notification :as notification]
+            [frontend.handler.external :as external-handler]))
+
+(rum/defc import-cp < rum/reactive
+  []
+  [:div#import
+   [:h1.title "Import JSON from Roam Research"]
+   [:input
+    {:id "import-roam"
+     :type "file"
+     :on-change (fn [e]
+                  (let [file (first (array-seq (.-files (.-target e))))
+                        file-name (gobj/get file "name")]
+                    (if (string/ends-with? file-name ".json")
+                      (let [reader (js/FileReader.)]
+                        (set! (.-onload reader)
+                              (fn [e]
+                                (let [text (.. e -target -result)]
+                                  (external-handler/import-from-roam-json! text))))
+                        (.readAsText reader file))
+                      (notification/show! "Please choose a JSON file."
+                                          :error))))
+     }]
+
+   ;; TODO: import status process
+   ])

web/src/main/frontend/components/sidebar.cljs

@@ -273,6 +273,9 @@
                (when current-repo
                  {:title "Settings"
                   :options {:href "/settings"}})
+               (when current-repo
+                 {:title "Import"
+                  :options {:href "/import"}})
                {:title [:div.flex-row.flex.justify-between.items-center
                         [:span "Join the community"]
                         svg/discord]

web/src/main/frontend/external.cljs

@@ -0,0 +1,17 @@
+(ns frontend.external
+  (:require [frontend.external.roam :refer [->Roam]]
+            [frontend.external.protocol :as protocol]))
+
+(defonce roam-record (->Roam))
+
+(defn get-record
+  [type]
+  (case type
+    :roam
+    roam-record
+    nil))
+
+(defn to-markdown-files
+  [type content config]
+  (when-let [record (get-record (keyword type))]
+    (protocol/toMarkdownFiles record content config)))

web/src/main/frontend/external/roam.cljs

@@ -2,38 +2,115 @@
   (:require [frontend.external.protocol :as protocol]
             [cljs-bean.core :as bean]
             [medley.core :as medley]
-            [clojure.walk :as walk]))
+            [clojure.walk :as walk]
+            [clojure.string :as string]
+            [frontend.util :as util]))
 
+(defonce all-refed-uids (atom #{}))
 (defonce uid->uuid (atom {}))
 
+(defn- reset-state!
+  []
+  (reset! all-refed-uids #{})
+  (reset! uid->uuid {}))
+
+;; DONE: 1. uid converted to a uuid
+;; DONE: 2. merge pages with same names (case-sensitive)
+;; DONE: 3. mldoc add support to roam research macros, or we can transform here.
+;; TODO: 4. mldoc add support to nested links
+;; TODO: hiccup
+
+(defonce uid-pattern #"\(\(([a-zA-Z0-9_\\-]{9})\)\)")
+(defonce macro-pattern #"\{\{([^{}]+)\}\}")
+
+(defn uid-transform
+  [text]
+  (string/replace text uid-pattern (fn [[_ uid]]
+                                     (let [id (get @uid->uuid uid uid)]
+                                       (str "((" id "))")))))
+
+(defn macro-transform
+  [text]
+  (string/replace text macro-pattern (fn [[original text]]
+                                       (let [[name arg] (string/split text #": ")]
+                                         (if name
+                                           (let [name (case name
+                                                        "[[embed]]" "embed"
+                                                        name)]
+                                             (util/format "{{{%s %s}}}" name arg))
+                                           original)))))
+
+(defn load-all-refed-uids!
+  [data]
+  (let [full-text (atom "")]
+    (walk/postwalk
+     (fn [f]
+       (when (and (map? f) (:string f))
+         (swap! full-text (fn [v] (str v (:string f)))))
+       f)
+     data)
+    (let [uids (->> (re-seq uid-pattern @full-text)
+                    (map last)
+                    (distinct)
+                    (set))]
+      (reset! all-refed-uids uids))))
+
+(defn transform
+  [text]
+  (-> text
+      (string/replace "{{[[TODO]]}}" "TODO")
+      (string/replace "{{[[DONE]]}}" "DONE")
+      (uid-transform)
+      (macro-transform)))
+
+;; #"(([a-zA-Z0-9_\\-]{9}))"
 (declare children->text)
 (defn child->text
-  [{:keys [uid string children] :as child}]
+  [{:keys [uid string children] :as child} level]
-  (when-not (get @uid->uuid uid)
+  (when-not (and (get @uid->uuid uid) uid)
     (swap! uid->uuid assoc uid (medley/random-uuid)))
-  (let [children-text (->> (map children->text children)
+  (let [children-text (children->text children (inc level))
-                           (interpose "\n")
+        level-pattern (apply str (repeat level "#"))
-                           (apply str))]
+        properties (when (contains? @all-refed-uids uid)
+                     (str
+                      (util/format ":PROPERTIES:\n:CUSTOM_ID:%s\n:END:"
+                                   (str (get @uid->uuid uid)))
+                      "\n"))]
     (if string
-      (str string "\n" children-text)
+      (str level-pattern " " (string/triml string) "\n" properties children-text)
       children-text)))
 
 (defn children->text
-  [children]
+  [children level]
-  (map child->text children))
+  (->> (map #(child->text % level) children)
+       (interpose "\n")
+       (apply str)))
 
 (defn ->file
   [page-data]
-  (let [{:keys [create-time title children edit-time]} page-data]
+  (let [{:keys [create-time title children edit-time]} page-data
+        initial-level 2]
     {:title title
      :created-at create-time
      :last-modified-at edit-time
-     :text (children->text children)}))
+     :text (when-let [text (children->text children initial-level)]
+             (let [front-matter (util/format "---\ntitle: %s\n---\n\n" title)]
+               (str front-matter (transform text))))}))
 
 (defn ->files
   [edn-data]
-  (let [pages-with-data (filter :children edn-data)]
+  (load-all-refed-uids! edn-data)
-    (map ->file pages-with-data)))
+  (let [pages-with-data (filter :children edn-data)
+        files (map ->file edn-data)
+        files (group-by (fn [f] (string/lower-case (:title f)))
+                        files)]
+    (map
+      (fn [[_ [fst & others]]]
+        (assoc fst :text
+               (->> (map :text (cons fst others))
+                    (interpose "\n")
+                    (apply str))))
+      files)))
 
 (defrecord Roam []
   protocol/External
@@ -41,7 +118,6 @@
     (let [data (bean/->clj (js/JSON.parse content))]
       (->files data))))
 
-;; (:create-email :create-time :title :children :edit-time :edit-email)
+(comment
-(defonce test-roam-json (frontend.db/get-file "same.json"))
+  (defonce test-roam-json (frontend.db/get-file "same.json"))
-
+  (defonce edn-data (bean/->clj (js/JSON.parse test-roam-json))))
-(defonce edn-data (bean/->clj (js/JSON.parse test-roam-json)))

web/src/main/frontend/handler/external.cljs

@@ -0,0 +1,21 @@
+(ns frontend.handler.external
+  (:require [frontend.external :as external]
+            [frontend.handler.file :as file-handler]
+            [frontend.handler.notification :as notification]
+            [frontend.state :as state]
+            [clojure.string :as string]))
+
+(defn import-from-roam-json!
+  [data]
+  (when-let [repo (state/get-current-repo)]
+    (let [files (external/to-markdown-files :roam data {})]
+     (doseq [file files]
+       (try
+         (when-let [text (:text file)]
+           (let [path (str "pages/" (string/replace (:title file) "/" "-") ".md")]
+             (file-handler/alter-file repo path text {})))
+         (catch js/Error e
+           (let [message (str "File " (:title file) " imported failed.")]
+             (println message)
+             (js/console.error e)
+             (notification/show! message :error))))))))

web/src/main/frontend/handler/repo.cljs

@@ -427,7 +427,8 @@
   (periodically-pull repo-url pull-now?)
   (when (and
          (or (not config/dev?)
-             (= repo-url "https://github.com/tiensonqin/empty-repo"))
+             ;; (= repo-url "https://github.com/tiensonqin/empty-repo")
+             )
          (not (false? (:git-auto-push (state/get-config repo-url)))))
     (periodically-push-tasks repo-url)))
 

web/src/main/frontend/handler/route.cljs

@@ -52,6 +52,8 @@
     "Draw"
     :settings
     "Settings"
+    :import
+    "Import data into Logseq"
     "Logseq"))
 
 (defn set-route-match!

web/src/main/frontend/routes.cljs

@@ -6,7 +6,8 @@
             [frontend.components.page :as page]
             [frontend.components.diff :as diff]
             [frontend.components.draw :as draw]
-            [frontend.components.settings :as settings]))
+            [frontend.components.settings :as settings]
+            [frontend.components.external :as external]))
 
 (def routes
   [["/"
@@ -55,4 +56,8 @@
 
    ["/settings"
     {:name :settings
-     :view settings/settings}]])
+     :view settings/settings}]
+
+   ["/import"
+    {:name :import
+     :view external/import-cp}]])