From b9bbd0c6056d170c7e04c555030a9ebcf9c7a062 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Fri, 13 May 2022 15:47:54 -0400 Subject: [PATCH] Finish migrating extract to nbb --- .clj-kondo/config.edn | 1 + src/main/frontend/format/block.cljs | 70 ------------------- src/main/frontend/fs/watcher_handler.cljs | 2 +- src/main/frontend/handler/editor.cljs | 4 +- src/main/frontend/handler/file.cljs | 14 ++-- src/main/logseq/graph_parser/block.cljc | 70 +++++++++++++++++++ .../graph_parser/extract.cljc} | 31 ++++---- .../frontend/modules/outliner/core_test.cljs | 4 +- .../graph_parser}/extract_test.cljs | 10 +-- .../logseq/graph_parser/nbb_test_runner.cljs | 6 +- 10 files changed, 110 insertions(+), 102 deletions(-) rename src/main/{frontend/handler/extract.cljs => logseq/graph_parser/extract.cljc} (89%) rename src/test/{frontend/handler => logseq/graph_parser}/extract_test.cljs (76%) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index ca3f6d717..c0eb35060 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -23,6 +23,7 @@ frontend.config config frontend.format.mldoc mldoc frontend.format.block block + frontend.handler.extract extract logseq.graph-parser.text text logseq.graph-parser.block gp-block logseq.graph-parser.mldoc gp-mldoc diff --git a/src/main/frontend/format/block.cljs b/src/main/frontend/format/block.cljs index 4d2ff354d..7f20ea7b7 100644 --- a/src/main/frontend/format/block.cljs +++ b/src/main/frontend/format/block.cljs @@ -26,76 +26,6 @@ ([original-page-name with-id? with-timestamp?] (gp-block/page-name->map original-page-name with-id? (db/get-db (state/get-current-repo)) with-timestamp? (state/get-date-formatter)))) -(defn with-parent-and-left - [page-id blocks] - (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks) - parents [{:page/id page-id ; db id or a map {:block/name "xxx"} - :block/level 0 - :block/level-spaces 0}] - result []] - (if (empty? blocks) - (map #(dissoc % :block/level-spaces) result) - (let [[block & others] blocks - level-spaces (:block/level-spaces block) - {:block/keys [uuid level parent] :as last-parent} (last parents) - parent-spaces (:block/level-spaces last-parent) - [blocks parents result] - (cond - (= level-spaces parent-spaces) ; sibling - (let [block (assoc block - :block/parent parent - :block/left [:block/uuid uuid] - :block/level level) - parents' (conj (vec (butlast parents)) block) - result' (conj result block)] - [others parents' result']) - - (> level-spaces parent-spaces) ; child - (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent)) - block (cond-> - (assoc block - :block/parent parent - :block/left parent) - ;; fix block levels with wrong order - ;; For example: - ;; - a - ;; - b - ;; What if the input indentation is two spaces instead of 4 spaces - (>= (- level-spaces parent-spaces) 1) - (assoc :block/level (inc level))) - parents' (conj parents block) - result' (conj result block)] - [others parents' result']) - - (< level-spaces parent-spaces) - (cond - (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent - (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)) - left (last parents') - blocks (cons (assoc (first blocks) - :block/level (dec level) - :block/left [:block/uuid (:block/uuid left)]) - (rest blocks))] - [blocks parents' result]) - - :else - (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents) - left (first r) - parent-id (if-let [block-id (:block/uuid (last f))] - [:block/uuid block-id] - page-id) - block (cond-> - (assoc block - :block/parent parent-id - :block/left [:block/uuid (:block/uuid left)] - :block/level (:block/level left) - :block/level-spaces (:block/level-spaces left))) - - parents' (->> (concat f [block]) vec) - result' (conj result block)] - [others parents' result'])))] - (recur blocks parents result))))) - (defn parse-block ([block] (parse-block block nil)) diff --git a/src/main/frontend/fs/watcher_handler.cljs b/src/main/frontend/fs/watcher_handler.cljs index 195f0b170..ca6735f44 100644 --- a/src/main/frontend/fs/watcher_handler.cljs +++ b/src/main/frontend/fs/watcher_handler.cljs @@ -4,7 +4,7 @@ [frontend.db :as db] [frontend.db.model :as model] [frontend.handler.editor :as editor] - [frontend.handler.extract :as extract] + [logseq.graph-parser.extract :as extract] [frontend.handler.file :as file-handler] [frontend.handler.page :as page-handler] [frontend.handler.repo :as repo-handler] diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index de151363b..ac24c325c 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -1960,7 +1960,7 @@ (let [blocks (block-tree->blocks tree-vec format) target-block (db/pull target-block-id) page-id (:db/id (:block/page target-block)) - blocks (block/with-parent-and-left page-id blocks)] + blocks (gp-block/with-parent-and-left page-id blocks)] (paste-blocks blocks {:target-block target-block @@ -2857,7 +2857,7 @@ (let [page-id (:db/id (:block/page editing-block)) blocks (block/extract-blocks (mldoc/->edn text (gp-mldoc/default-config format)) text true format) - blocks' (block/with-parent-and-left page-id blocks)] + blocks' (gp-block/with-parent-and-left page-id blocks)] (paste-blocks blocks' {})))) (defn- paste-segmented-text diff --git a/src/main/frontend/handler/file.cljs b/src/main/frontend/handler/file.cljs index e6b3b112c..a6a374243 100644 --- a/src/main/frontend/handler/file.cljs +++ b/src/main/frontend/handler/file.cljs @@ -11,7 +11,7 @@ [frontend.fs :as fs] [frontend.fs.nfs :as nfs] [frontend.handler.common :as common-handler] - [frontend.handler.extract :as extract-handler] + [logseq.graph-parser.extract :as extract] [frontend.handler.ui :as ui-handler] [frontend.state :as state] [frontend.util :as util] @@ -123,11 +123,15 @@ file-content [{:file/path file}] tx (if (contains? gp-config/mldoc-support-formats format) (let [[pages blocks] - (extract-handler/extract-blocks-pages - repo-url file content + (extract/extract-blocks-pages + file + content {:user-config (state/get-config) :date-formatter (state/get-date-formatter) - :page-name-order (state/page-name-order)}) + :page-name-order (state/page-name-order) + :block-pattern (config/get-block-pattern format) + :supported-formats (config/supported-formats) + :db (db/get-db (state/get-current-repo))}) first-page (first pages) delete-blocks (-> (concat @@ -149,7 +153,7 @@ (seq)) ;; To prevent "unique constraint" on datascript block-ids (set/union (set block-ids) (set block-refs-ids)) - pages (extract-handler/with-ref-pages pages blocks) + pages (extract/with-ref-pages pages blocks) pages-index (map #(select-keys % [:block/name]) pages)] ;; does order matter? (concat file-content pages-index delete-blocks pages block-ids blocks)) diff --git a/src/main/logseq/graph_parser/block.cljc b/src/main/logseq/graph_parser/block.cljc index c02cda92c..7cc7d4372 100644 --- a/src/main/logseq/graph_parser/block.cljc +++ b/src/main/logseq/graph_parser/block.cljc @@ -581,3 +581,73 @@ (map #(dissoc % :block/meta) result)) (catch :default e (log/error :extract-blocks-failure e)))) + +(defn with-parent-and-left + [page-id blocks] + (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks) + parents [{:page/id page-id ; db id or a map {:block/name "xxx"} + :block/level 0 + :block/level-spaces 0}] + result []] + (if (empty? blocks) + (map #(dissoc % :block/level-spaces) result) + (let [[block & others] blocks + level-spaces (:block/level-spaces block) + {:block/keys [uuid level parent] :as last-parent} (last parents) + parent-spaces (:block/level-spaces last-parent) + [blocks parents result] + (cond + (= level-spaces parent-spaces) ; sibling + (let [block (assoc block + :block/parent parent + :block/left [:block/uuid uuid] + :block/level level) + parents' (conj (vec (butlast parents)) block) + result' (conj result block)] + [others parents' result']) + + (> level-spaces parent-spaces) ; child + (let [parent (if uuid [:block/uuid uuid] (:page/id last-parent)) + block (cond-> + (assoc block + :block/parent parent + :block/left parent) + ;; fix block levels with wrong order + ;; For example: + ;; - a + ;; - b + ;; What if the input indentation is two spaces instead of 4 spaces + (>= (- level-spaces parent-spaces) 1) + (assoc :block/level (inc level))) + parents' (conj parents block) + result' (conj result block)] + [others parents' result']) + + (< level-spaces parent-spaces) + (cond + (some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent + (let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)) + left (last parents') + blocks (cons (assoc (first blocks) + :block/level (dec level) + :block/left [:block/uuid (:block/uuid left)]) + (rest blocks))] + [blocks parents' result]) + + :else + (let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents) + left (first r) + parent-id (if-let [block-id (:block/uuid (last f))] + [:block/uuid block-id] + page-id) + block (cond-> + (assoc block + :block/parent parent-id + :block/left [:block/uuid (:block/uuid left)] + :block/level (:block/level left) + :block/level-spaces (:block/level-spaces left))) + + parents' (->> (concat f [block]) vec) + result' (conj result block)] + [others parents' result'])))] + (recur blocks parents result))))) diff --git a/src/main/frontend/handler/extract.cljs b/src/main/logseq/graph_parser/extract.cljc similarity index 89% rename from src/main/frontend/handler/extract.cljs rename to src/main/logseq/graph_parser/extract.cljc index 49b3c3b64..36d25f308 100644 --- a/src/main/frontend/handler/extract.cljs +++ b/src/main/logseq/graph_parser/extract.cljc @@ -1,17 +1,19 @@ -(ns frontend.handler.extract - "Extract helper." +(ns ^:nbb-compatible logseq.graph-parser.extract + ;; Disable clj linters since we don't support clj + #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off} + :unresolved-symbol {:level :off}}}}) (:require [clojure.set :as set] [clojure.string :as string] [clojure.walk :as walk] [datascript.core :as d] - [frontend.format.block :as block] [logseq.graph-parser.text :as text] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.block :as gp-block] [logseq.graph-parser.property :as gp-property] [logseq.graph-parser.config :as gp-config] - [lambdaisland.glogi :as log])) + #?(:org.babashka/nbb [logseq.graph-parser.log :as log] + :default [lambdaisland.glogi :as log]))) (defn- get-page-name [file ast page-name-order] @@ -40,13 +42,13 @@ ;; TODO: performance improvement (defn- extract-pages-and-blocks - #_:clj-kondo/ignore - [repo-url format ast properties file content date-formatter page-name-order] + [format ast properties file content {:keys [date-formatter page-name-order db] :as options}] (try + #_:clj-kondo/ignore ;;clj-kondo bug (let [page (get-page-name file ast page-name-order) [_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter) - blocks (->> (block/extract-blocks ast content false format) - (block/with-parent-and-left {:block/name page-name})) + blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order)) + (gp-block/with-parent-and-left {:block/name page-name})) ref-pages (atom #{}) ref-tags (atom #{}) blocks (map (fn [block] @@ -91,7 +93,7 @@ (cond-> (gp-util/remove-nils (assoc - (block/page-name->map page false) + (gp-block/page-name->map page false db true date-formatter) :block/file {:file/path (gp-util/path-normalize file)})) (seq properties) (assoc :block/properties properties) @@ -111,7 +113,7 @@ (when (text/namespace-page? page) (->> (gp-util/split-namespace-pages page) (map (fn [page] - (-> (block/page-name->map page true) + (-> (gp-block/page-name->map page true db true date-formatter) (assoc :block/format format))))))) pages (->> (concat [page-entity] @@ -131,11 +133,11 @@ blocks (->> (remove nil? blocks) (map (fn [b] (dissoc b :block/title :block/body :block/level :block/children :block/meta :block/anchor))))] [pages blocks]) - (catch js/Error e + (catch :default e (log/error :exception e)))) (defn extract-blocks-pages - [repo-url file content {:keys [user-config date-formatter page-name-order]}] + [file content {:keys [user-config] :as options}] (if (string/blank? content) [] (let [format (gp-util/get-format file) @@ -161,11 +163,10 @@ (string/replace (or v "") "\\" ""))) properties)))] (extract-pages-and-blocks - repo-url format ast properties - file content date-formatter page-name-order))))) + file content options))))) -(defn with-block-uuid +(defn- with-block-uuid [pages] (->> (gp-util/distinct-by :block/name pages) (map (fn [page] diff --git a/src/test/frontend/modules/outliner/core_test.cljs b/src/test/frontend/modules/outliner/core_test.cljs index 072007201..59a2b1c6a 100644 --- a/src/test/frontend/modules/outliner/core_test.cljs +++ b/src/test/frontend/modules/outliner/core_test.cljs @@ -8,7 +8,7 @@ [frontend.db :as db] [frontend.db.model :as db-model] [clojure.walk :as walk] - [frontend.format.block :as block] + [logseq.graph-parser.block :as gp-block] [datascript.core :as d] [frontend.test.helper :as helper])) @@ -65,7 +65,7 @@ (defn- build-blocks [tree] - (block/with-parent-and-left 1 (build-node-tree tree))) + (gp-block/with-parent-and-left 1 (build-node-tree tree))) (defn transact-tree! [tree] diff --git a/src/test/frontend/handler/extract_test.cljs b/src/test/logseq/graph_parser/extract_test.cljs similarity index 76% rename from src/test/frontend/handler/extract_test.cljs rename to src/test/logseq/graph_parser/extract_test.cljs index 7f9b21e9f..dc00d1647 100644 --- a/src/test/frontend/handler/extract_test.cljs +++ b/src/test/logseq/graph_parser/extract_test.cljs @@ -1,17 +1,17 @@ -(ns frontend.handler.extract-test +(ns logseq.graph-parser.extract-test (:require [cljs.test :refer [async deftest is]] - [frontend.handler.extract :as extract] - [frontend.util :as util] + [logseq.graph-parser.extract :as extract] + [clojure.pprint :as pprint] [promesa.core :as p])) (defn- extract [text] - (p/let [result (extract/extract-blocks-pages "repo" "a.md" text {}) + (p/let [result (extract/extract-blocks-pages "a.md" text {:block-pattern "-"}) result (last result) lefts (map (juxt :block/parent :block/left) result)] (if (not= (count lefts) (count (distinct lefts))) (do - (util/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result)) + (pprint/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result)) (throw (js/Error. ":block/parent && :block/left conflicts"))) (mapv :block/content result)))) diff --git a/src/test/logseq/graph_parser/nbb_test_runner.cljs b/src/test/logseq/graph_parser/nbb_test_runner.cljs index 85c1a6365..0f8ac4868 100644 --- a/src/test/logseq/graph_parser/nbb_test_runner.cljs +++ b/src/test/logseq/graph_parser/nbb_test_runner.cljs @@ -6,7 +6,8 @@ [logseq.graph-parser.text-test] [logseq.graph-parser.mldoc-test] [logseq.graph-parser.block-test] - [logseq.graph-parser.property-test])) + [logseq.graph-parser.property-test] + [logseq.graph-parser.extract-test])) (defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m] (when-not (cljs.test/successful? m) @@ -21,4 +22,5 @@ (t/run-tests 'logseq.graph-parser.mldoc-test 'logseq.graph-parser.text-test 'logseq.graph-parser.property-test - 'logseq.graph-parser.block-test)) + 'logseq.graph-parser.block-test + 'logseq.graph-parser.extract-test))