From 9fc3f0f21dcffa766a7629f00096c9b2eccf435c Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 20 Aug 2020 00:00:05 +0800 Subject: [PATCH] Roam data import prototype --- .../main/frontend/components/external.cljs | 30 +++++ web/src/main/frontend/components/sidebar.cljs | 3 + web/src/main/frontend/external.cljs | 17 +++ web/src/main/frontend/external/roam.cljs | 110 +++++++++++++++--- web/src/main/frontend/handler/external.cljs | 21 ++++ web/src/main/frontend/handler/repo.cljs | 3 +- web/src/main/frontend/handler/route.cljs | 2 + web/src/main/frontend/routes.cljs | 9 +- 8 files changed, 175 insertions(+), 20 deletions(-) create mode 100644 web/src/main/frontend/components/external.cljs create mode 100644 web/src/main/frontend/external.cljs create mode 100644 web/src/main/frontend/handler/external.cljs diff --git a/web/src/main/frontend/components/external.cljs b/web/src/main/frontend/components/external.cljs new file mode 100644 index 000000000..c98e88265 --- /dev/null +++ b/web/src/main/frontend/components/external.cljs @@ -0,0 +1,30 @@ +(ns frontend.components.external + (:require [rum.core :as rum] + [goog.object :as gobj] + [clojure.string :as string] + [frontend.handler.notification :as notification] + [frontend.handler.external :as external-handler])) + +(rum/defc import-cp < rum/reactive + [] + [:div#import + [:h1.title "Import JSON from Roam Research"] + [:input + {:id "import-roam" + :type "file" + :on-change (fn [e] + (let [file (first (array-seq (.-files (.-target e)))) + file-name (gobj/get file "name")] + (if (string/ends-with? file-name ".json") + (let [reader (js/FileReader.)] + (set! (.-onload reader) + (fn [e] + (let [text (.. e -target -result)] + (external-handler/import-from-roam-json! text)))) + (.readAsText reader file)) + (notification/show! "Please choose a JSON file." + :error)))) + }] + + ;; TODO: import status process + ]) diff --git a/web/src/main/frontend/components/sidebar.cljs b/web/src/main/frontend/components/sidebar.cljs index d311d1f93..793f3d5df 100644 --- a/web/src/main/frontend/components/sidebar.cljs +++ b/web/src/main/frontend/components/sidebar.cljs @@ -273,6 +273,9 @@ (when current-repo {:title "Settings" :options {:href "/settings"}}) + (when current-repo + {:title "Import" + :options {:href "/import"}}) {:title [:div.flex-row.flex.justify-between.items-center [:span "Join the community"] svg/discord] diff --git a/web/src/main/frontend/external.cljs b/web/src/main/frontend/external.cljs new file mode 100644 index 000000000..0c8523d9b --- /dev/null +++ b/web/src/main/frontend/external.cljs @@ -0,0 +1,17 @@ +(ns frontend.external + (:require [frontend.external.roam :refer [->Roam]] + [frontend.external.protocol :as protocol])) + +(defonce roam-record (->Roam)) + +(defn get-record + [type] + (case type + :roam + roam-record + nil)) + +(defn to-markdown-files + [type content config] + (when-let [record (get-record (keyword type))] + (protocol/toMarkdownFiles record content config))) diff --git a/web/src/main/frontend/external/roam.cljs b/web/src/main/frontend/external/roam.cljs index adf23e8e7..f40502ce1 100644 --- a/web/src/main/frontend/external/roam.cljs +++ b/web/src/main/frontend/external/roam.cljs @@ -2,38 +2,115 @@ (:require [frontend.external.protocol :as protocol] [cljs-bean.core :as bean] [medley.core :as medley] - [clojure.walk :as walk])) + [clojure.walk :as walk] + [clojure.string :as string] + [frontend.util :as util])) +(defonce all-refed-uids (atom #{})) (defonce uid->uuid (atom {})) +(defn- reset-state! + [] + (reset! all-refed-uids #{}) + (reset! uid->uuid {})) + +;; DONE: 1. uid converted to a uuid +;; DONE: 2. merge pages with same names (case-sensitive) +;; DONE: 3. mldoc add support to roam research macros, or we can transform here. +;; TODO: 4. mldoc add support to nested links +;; TODO: hiccup + +(defonce uid-pattern #"\(\(([a-zA-Z0-9_\\-]{9})\)\)") +(defonce macro-pattern #"\{\{([^{}]+)\}\}") + +(defn uid-transform + [text] + (string/replace text uid-pattern (fn [[_ uid]] + (let [id (get @uid->uuid uid uid)] + (str "((" id "))"))))) + +(defn macro-transform + [text] + (string/replace text macro-pattern (fn [[original text]] + (let [[name arg] (string/split text #": ")] + (if name + (let [name (case name + "[[embed]]" "embed" + name)] + (util/format "{{{%s %s}}}" name arg)) + original))))) + +(defn load-all-refed-uids! + [data] + (let [full-text (atom "")] + (walk/postwalk + (fn [f] + (when (and (map? f) (:string f)) + (swap! full-text (fn [v] (str v (:string f))))) + f) + data) + (let [uids (->> (re-seq uid-pattern @full-text) + (map last) + (distinct) + (set))] + (reset! all-refed-uids uids)))) + +(defn transform + [text] + (-> text + (string/replace "{{[[TODO]]}}" "TODO") + (string/replace "{{[[DONE]]}}" "DONE") + (uid-transform) + (macro-transform))) + +;; #"(([a-zA-Z0-9_\\-]{9}))" (declare children->text) (defn child->text - [{:keys [uid string children] :as child}] - (when-not (get @uid->uuid uid) + [{:keys [uid string children] :as child} level] + (when-not (and (get @uid->uuid uid) uid) (swap! uid->uuid assoc uid (medley/random-uuid))) - (let [children-text (->> (map children->text children) - (interpose "\n") - (apply str))] + (let [children-text (children->text children (inc level)) + level-pattern (apply str (repeat level "#")) + properties (when (contains? @all-refed-uids uid) + (str + (util/format ":PROPERTIES:\n:CUSTOM_ID:%s\n:END:" + (str (get @uid->uuid uid))) + "\n"))] (if string - (str string "\n" children-text) + (str level-pattern " " (string/triml string) "\n" properties children-text) children-text))) (defn children->text - [children] - (map child->text children)) + [children level] + (->> (map #(child->text % level) children) + (interpose "\n") + (apply str))) (defn ->file [page-data] - (let [{:keys [create-time title children edit-time]} page-data] + (let [{:keys [create-time title children edit-time]} page-data + initial-level 2] {:title title :created-at create-time :last-modified-at edit-time - :text (children->text children)})) + :text (when-let [text (children->text children initial-level)] + (let [front-matter (util/format "---\ntitle: %s\n---\n\n" title)] + (str front-matter (transform text))))})) (defn ->files [edn-data] - (let [pages-with-data (filter :children edn-data)] - (map ->file pages-with-data))) + (load-all-refed-uids! edn-data) + (let [pages-with-data (filter :children edn-data) + files (map ->file edn-data) + files (group-by (fn [f] (string/lower-case (:title f))) + files)] + (map + (fn [[_ [fst & others]]] + (assoc fst :text + (->> (map :text (cons fst others)) + (interpose "\n") + (apply str)))) + files))) (defrecord Roam [] protocol/External @@ -41,7 +118,6 @@ (let [data (bean/->clj (js/JSON.parse content))] (->files data)))) -;; (:create-email :create-time :title :children :edit-time :edit-email) -(defonce test-roam-json (frontend.db/get-file "same.json")) - -(defonce edn-data (bean/->clj (js/JSON.parse test-roam-json))) +(comment + (defonce test-roam-json (frontend.db/get-file "same.json")) + (defonce edn-data (bean/->clj (js/JSON.parse test-roam-json)))) diff --git a/web/src/main/frontend/handler/external.cljs b/web/src/main/frontend/handler/external.cljs new file mode 100644 index 000000000..c0f64b0c3 --- /dev/null +++ b/web/src/main/frontend/handler/external.cljs @@ -0,0 +1,21 @@ +(ns frontend.handler.external + (:require [frontend.external :as external] + [frontend.handler.file :as file-handler] + [frontend.handler.notification :as notification] + [frontend.state :as state] + [clojure.string :as string])) + +(defn import-from-roam-json! + [data] + (when-let [repo (state/get-current-repo)] + (let [files (external/to-markdown-files :roam data {})] + (doseq [file files] + (try + (when-let [text (:text file)] + (let [path (str "pages/" (string/replace (:title file) "/" "-") ".md")] + (file-handler/alter-file repo path text {}))) + (catch js/Error e + (let [message (str "File " (:title file) " imported failed.")] + (println message) + (js/console.error e) + (notification/show! message :error)))))))) diff --git a/web/src/main/frontend/handler/repo.cljs b/web/src/main/frontend/handler/repo.cljs index 7d1848084..5c16bc2a3 100644 --- a/web/src/main/frontend/handler/repo.cljs +++ b/web/src/main/frontend/handler/repo.cljs @@ -427,7 +427,8 @@ (periodically-pull repo-url pull-now?) (when (and (or (not config/dev?) - (= repo-url "https://github.com/tiensonqin/empty-repo")) + ;; (= repo-url "https://github.com/tiensonqin/empty-repo") + ) (not (false? (:git-auto-push (state/get-config repo-url))))) (periodically-push-tasks repo-url))) diff --git a/web/src/main/frontend/handler/route.cljs b/web/src/main/frontend/handler/route.cljs index 0a4cf0ae6..03995b567 100644 --- a/web/src/main/frontend/handler/route.cljs +++ b/web/src/main/frontend/handler/route.cljs @@ -52,6 +52,8 @@ "Draw" :settings "Settings" + :import + "Import data into Logseq" "Logseq")) (defn set-route-match! diff --git a/web/src/main/frontend/routes.cljs b/web/src/main/frontend/routes.cljs index bfe117d4a..2e4876008 100644 --- a/web/src/main/frontend/routes.cljs +++ b/web/src/main/frontend/routes.cljs @@ -6,7 +6,8 @@ [frontend.components.page :as page] [frontend.components.diff :as diff] [frontend.components.draw :as draw] - [frontend.components.settings :as settings])) + [frontend.components.settings :as settings] + [frontend.components.external :as external])) (def routes [["/" @@ -55,4 +56,8 @@ ["/settings" {:name :settings - :view settings/settings}]]) + :view settings/settings}] + + ["/import" + {:name :import + :view external/import-cp}]])