greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Finish migrating extract to nbb

pull/5299/head
Gabriel Horner 2022-05-13 15:47:54 -04:00
parent a832564782
commit b9bbd0c605
10 changed files with 110 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.clj-kondo/config.edn
View File

@ -23,6 +23,7 @@
frontend.config config
frontend.format.mldoc mldoc
frontend.format.block block
frontend.handler.extract extract
logseq.graph-parser.text text
logseq.graph-parser.block gp-block
logseq.graph-parser.mldoc gp-mldoc

70
src/main/frontend/format/block.cljs
View File

@ -26,76 +26,6 @@
([original-page-name with-id? with-timestamp?]
(gp-block/page-name->map original-page-name with-id? (db/get-db (state/get-current-repo)) with-timestamp? (state/get-date-formatter))))
(defn with-parent-and-left
[page-id blocks]
(loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
parents [{:page/id page-id ; db id or a map {:block/name "xxx"}
:block/level 0
:block/level-spaces 0}]
result []]
(if (empty? blocks)
(map #(dissoc % :block/level-spaces) result)
(let [[block & others] blocks
level-spaces (:block/level-spaces block)
{:block/keys [uuid level parent] :as last-parent} (last parents)
parent-spaces (:block/level-spaces last-parent)
[blocks parents result]
(cond
(= level-spaces parent-spaces) ; sibling
(let [block (assoc block
:block/parent parent
:block/left [:block/uuid uuid]
:block/level level)
parents' (conj (vec (butlast parents)) block)
result' (conj result block)]
[others parents' result'])
(> level-spaces parent-spaces) ; child
(let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
block (cond->
(assoc block
:block/parent parent
:block/left parent)
;; fix block levels with wrong order
;; For example:
;; - a
;; - b
;; What if the input indentation is two spaces instead of 4 spaces
(>= (- level-spaces parent-spaces) 1)
(assoc :block/level (inc level)))
parents' (conj parents block)
result' (conj result block)]
[others parents' result'])
(< level-spaces parent-spaces)
(cond
(some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
(let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
left (last parents')
blocks (cons (assoc (first blocks)
:block/level (dec level)
:block/left [:block/uuid (:block/uuid left)])
(rest blocks))]
[blocks parents' result])
:else
(let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
left (first r)
parent-id (if-let [block-id (:block/uuid (last f))]
[:block/uuid block-id]
page-id)
block (cond->
(assoc block
:block/parent parent-id
:block/left [:block/uuid (:block/uuid left)]
:block/level (:block/level left)
:block/level-spaces (:block/level-spaces left)))
parents' (->> (concat f [block]) vec)
result' (conj result block)]
[others parents' result'])))]
(recur blocks parents result)))))
(defn parse-block
([block]
(parse-block block nil))

2
src/main/frontend/fs/watcher_handler.cljs
View File

@ -4,7 +4,7 @@
[frontend.db :as db]
[frontend.db.model :as model]
[frontend.handler.editor :as editor]
[frontend.handler.extract :as extract]
[logseq.graph-parser.extract :as extract]
[frontend.handler.file :as file-handler]
[frontend.handler.page :as page-handler]
[frontend.handler.repo :as repo-handler]

4
src/main/frontend/handler/editor.cljs
View File

@ -1960,7 +1960,7 @@
(let [blocks (block-tree->blocks tree-vec format)
target-block (db/pull target-block-id)
page-id (:db/id (:block/page target-block))
blocks (block/with-parent-and-left page-id blocks)]
blocks (gp-block/with-parent-and-left page-id blocks)]
(paste-blocks
blocks
{:target-block target-block
@ -2857,7 +2857,7 @@
(let [page-id (:db/id (:block/page editing-block))
blocks (block/extract-blocks
(mldoc/->edn text (gp-mldoc/default-config format)) text true format)
blocks' (block/with-parent-and-left page-id blocks)]
blocks' (gp-block/with-parent-and-left page-id blocks)]
(paste-blocks blocks' {}))))
(defn- paste-segmented-text

14
src/main/frontend/handler/file.cljs
View File

@ -11,7 +11,7 @@
[frontend.fs :as fs]
[frontend.fs.nfs :as nfs]
[frontend.handler.common :as common-handler]
[frontend.handler.extract :as extract-handler]
[logseq.graph-parser.extract :as extract]
[frontend.handler.ui :as ui-handler]
[frontend.state :as state]
[frontend.util :as util]
@ -123,11 +123,15 @@
file-content [{:file/path file}]
tx (if (contains? gp-config/mldoc-support-formats format)
(let [[pages blocks]
(extract-handler/extract-blocks-pages
repo-url file content
(extract/extract-blocks-pages
file
content
{:user-config (state/get-config)
:date-formatter (state/get-date-formatter)
:page-name-order (state/page-name-order)})
:page-name-order (state/page-name-order)
:block-pattern (config/get-block-pattern format)
:supported-formats (config/supported-formats)
:db (db/get-db (state/get-current-repo))})
first-page (first pages)
delete-blocks (->
(concat
@ -149,7 +153,7 @@
(seq))
;; To prevent "unique constraint" on datascript
block-ids (set/union (set block-ids) (set block-refs-ids))
pages (extract-handler/with-ref-pages pages blocks)
pages (extract/with-ref-pages pages blocks)
pages-index (map #(select-keys % [:block/name]) pages)]
;; does order matter?
(concat file-content pages-index delete-blocks pages block-ids blocks))

70
src/main/logseq/graph_parser/block.cljc
View File

@ -581,3 +581,73 @@
(map #(dissoc % :block/meta) result))
(catch :default e
(log/error :extract-blocks-failure e))))
(defn with-parent-and-left
[page-id blocks]
(loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
parents [{:page/id page-id ; db id or a map {:block/name "xxx"}
:block/level 0
:block/level-spaces 0}]
result []]
(if (empty? blocks)
(map #(dissoc % :block/level-spaces) result)
(let [[block & others] blocks
level-spaces (:block/level-spaces block)
{:block/keys [uuid level parent] :as last-parent} (last parents)
parent-spaces (:block/level-spaces last-parent)
[blocks parents result]
(cond
(= level-spaces parent-spaces) ; sibling
(let [block (assoc block
:block/parent parent
:block/left [:block/uuid uuid]
:block/level level)
parents' (conj (vec (butlast parents)) block)
result' (conj result block)]
[others parents' result'])
(> level-spaces parent-spaces) ; child
(let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
block (cond->
(assoc block
:block/parent parent
:block/left parent)
;; fix block levels with wrong order
;; For example:
;; - a
;; - b
;; What if the input indentation is two spaces instead of 4 spaces
(>= (- level-spaces parent-spaces) 1)
(assoc :block/level (inc level)))
parents' (conj parents block)
result' (conj result block)]
[others parents' result'])
(< level-spaces parent-spaces)
(cond
(some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
(let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
left (last parents')
blocks (cons (assoc (first blocks)
:block/level (dec level)
:block/left [:block/uuid (:block/uuid left)])
(rest blocks))]
[blocks parents' result])
:else
(let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
left (first r)
parent-id (if-let [block-id (:block/uuid (last f))]
[:block/uuid block-id]
page-id)
block (cond->
(assoc block
:block/parent parent-id
:block/left [:block/uuid (:block/uuid left)]
:block/level (:block/level left)
:block/level-spaces (:block/level-spaces left)))
parents' (->> (concat f [block]) vec)
result' (conj result block)]
[others parents' result'])))]
(recur blocks parents result)))))

31
src/main/frontend/handler/extract.cljs → src/main/logseq/graph_parser/extract.cljc
View File

@ -1,17 +1,19 @@
(ns frontend.handler.extract
"Extract helper."
(ns ^:nbb-compatible logseq.graph-parser.extract
;; Disable clj linters since we don't support clj
#?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
:unresolved-symbol {:level :off}}}})
(:require [clojure.set :as set]
[clojure.string :as string]
[clojure.walk :as walk]
[datascript.core :as d]
[frontend.format.block :as block]
[logseq.graph-parser.text :as text]
[logseq.graph-parser.util :as gp-util]
[logseq.graph-parser.mldoc :as gp-mldoc]
[logseq.graph-parser.block :as gp-block]
[logseq.graph-parser.property :as gp-property]
[logseq.graph-parser.config :as gp-config]
[lambdaisland.glogi :as log]))
#?(:org.babashka/nbb [logseq.graph-parser.log :as log]
:default [lambdaisland.glogi :as log])))
(defn- get-page-name
[file ast page-name-order]
@ -40,13 +42,13 @@
;; TODO: performance improvement
(defn- extract-pages-and-blocks
#_:clj-kondo/ignore
[repo-url format ast properties file content date-formatter page-name-order]
[format ast properties file content {:keys [date-formatter page-name-order db] :as options}]
(try
#_:clj-kondo/ignore ;;clj-kondo bug
(let [page (get-page-name file ast page-name-order)
[_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
blocks (->> (block/extract-blocks ast content false format)
(block/with-parent-and-left {:block/name page-name}))
blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order))
(gp-block/with-parent-and-left {:block/name page-name}))
ref-pages (atom #{})
ref-tags (atom #{})
blocks (map (fn [block]
@ -91,7 +93,7 @@
(cond->
(gp-util/remove-nils
(assoc
(block/page-name->map page false)
(gp-block/page-name->map page false db true date-formatter)
:block/file {:file/path (gp-util/path-normalize file)}))
(seq properties)
(assoc :block/properties properties)
@ -111,7 +113,7 @@
(when (text/namespace-page? page)
(->> (gp-util/split-namespace-pages page)
(map (fn [page]
(-> (block/page-name->map page true)
(-> (gp-block/page-name->map page true db true date-formatter)
(assoc :block/format format)))))))
pages (->> (concat
[page-entity]
@ -131,11 +133,11 @@
blocks (->> (remove nil? blocks)
(map (fn [b] (dissoc b :block/title :block/body :block/level :block/children :block/meta :block/anchor))))]
[pages blocks])
(catch js/Error e
(catch :default e
(log/error :exception e))))
(defn extract-blocks-pages
[repo-url file content {:keys [user-config date-formatter page-name-order]}]
[file content {:keys [user-config] :as options}]
(if (string/blank? content)
[]
(let [format (gp-util/get-format file)
@ -161,11 +163,10 @@
(string/replace (or v "") "\\" "")))
properties)))]
(extract-pages-and-blocks
repo-url
format ast properties
file content date-formatter page-name-order)))))
file content options)))))
(defn with-block-uuid
(defn- with-block-uuid
[pages]
(->> (gp-util/distinct-by :block/name pages)
(map (fn [page]

4
src/test/frontend/modules/outliner/core_test.cljs
View File

@ -8,7 +8,7 @@
[frontend.db :as db]
[frontend.db.model :as db-model]
[clojure.walk :as walk]
[frontend.format.block :as block]
[logseq.graph-parser.block :as gp-block]
[datascript.core :as d]
[frontend.test.helper :as helper]))
@ -65,7 +65,7 @@
(defn- build-blocks
[tree]
(block/with-parent-and-left 1 (build-node-tree tree)))
(gp-block/with-parent-and-left 1 (build-node-tree tree)))
(defn transact-tree!
[tree]

10
src/test/frontend/handler/extract_test.cljs → src/test/logseq/graph_parser/extract_test.cljs
View File

@ -1,17 +1,17 @@
(ns frontend.handler.extract-test
(ns logseq.graph-parser.extract-test
(:require [cljs.test :refer [async deftest is]]
[frontend.handler.extract :as extract]
[frontend.util :as util]
[logseq.graph-parser.extract :as extract]
[clojure.pprint :as pprint]
[promesa.core :as p]))
(defn- extract
[text]
(p/let [result (extract/extract-blocks-pages "repo" "a.md" text {})
(p/let [result (extract/extract-blocks-pages "a.md" text {:block-pattern "-"})
result (last result)
lefts (map (juxt :block/parent :block/left) result)]
(if (not= (count lefts) (count (distinct lefts)))
(do
(util/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
(pprint/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
(throw (js/Error. ":block/parent && :block/left conflicts")))
(mapv :block/content result))))

6
src/test/logseq/graph_parser/nbb_test_runner.cljs
View File

@ -6,7 +6,8 @@
[logseq.graph-parser.text-test]
[logseq.graph-parser.mldoc-test]
[logseq.graph-parser.block-test]
[logseq.graph-parser.property-test]))
[logseq.graph-parser.property-test]
[logseq.graph-parser.extract-test]))
(defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m]
(when-not (cljs.test/successful? m)
@ -21,4 +22,5 @@
(t/run-tests 'logseq.graph-parser.mldoc-test
'logseq.graph-parser.text-test
'logseq.graph-parser.property-test
'logseq.graph-parser.block-test))
'logseq.graph-parser.block-test
'logseq.graph-parser.extract-test))