mirror of https://github.com/logseq/logseq
Finish migrating extract to nbb
parent
a832564782
commit
b9bbd0c605
|
@ -23,6 +23,7 @@
|
|||
frontend.config config
|
||||
frontend.format.mldoc mldoc
|
||||
frontend.format.block block
|
||||
frontend.handler.extract extract
|
||||
logseq.graph-parser.text text
|
||||
logseq.graph-parser.block gp-block
|
||||
logseq.graph-parser.mldoc gp-mldoc
|
||||
|
|
|
@ -26,76 +26,6 @@
|
|||
([original-page-name with-id? with-timestamp?]
|
||||
(gp-block/page-name->map original-page-name with-id? (db/get-db (state/get-current-repo)) with-timestamp? (state/get-date-formatter))))
|
||||
|
||||
(defn with-parent-and-left
|
||||
[page-id blocks]
|
||||
(loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
|
||||
parents [{:page/id page-id ; db id or a map {:block/name "xxx"}
|
||||
:block/level 0
|
||||
:block/level-spaces 0}]
|
||||
result []]
|
||||
(if (empty? blocks)
|
||||
(map #(dissoc % :block/level-spaces) result)
|
||||
(let [[block & others] blocks
|
||||
level-spaces (:block/level-spaces block)
|
||||
{:block/keys [uuid level parent] :as last-parent} (last parents)
|
||||
parent-spaces (:block/level-spaces last-parent)
|
||||
[blocks parents result]
|
||||
(cond
|
||||
(= level-spaces parent-spaces) ; sibling
|
||||
(let [block (assoc block
|
||||
:block/parent parent
|
||||
:block/left [:block/uuid uuid]
|
||||
:block/level level)
|
||||
parents' (conj (vec (butlast parents)) block)
|
||||
result' (conj result block)]
|
||||
[others parents' result'])
|
||||
|
||||
(> level-spaces parent-spaces) ; child
|
||||
(let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
|
||||
block (cond->
|
||||
(assoc block
|
||||
:block/parent parent
|
||||
:block/left parent)
|
||||
;; fix block levels with wrong order
|
||||
;; For example:
|
||||
;; - a
|
||||
;; - b
|
||||
;; What if the input indentation is two spaces instead of 4 spaces
|
||||
(>= (- level-spaces parent-spaces) 1)
|
||||
(assoc :block/level (inc level)))
|
||||
parents' (conj parents block)
|
||||
result' (conj result block)]
|
||||
[others parents' result'])
|
||||
|
||||
(< level-spaces parent-spaces)
|
||||
(cond
|
||||
(some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
|
||||
(let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
|
||||
left (last parents')
|
||||
blocks (cons (assoc (first blocks)
|
||||
:block/level (dec level)
|
||||
:block/left [:block/uuid (:block/uuid left)])
|
||||
(rest blocks))]
|
||||
[blocks parents' result])
|
||||
|
||||
:else
|
||||
(let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
|
||||
left (first r)
|
||||
parent-id (if-let [block-id (:block/uuid (last f))]
|
||||
[:block/uuid block-id]
|
||||
page-id)
|
||||
block (cond->
|
||||
(assoc block
|
||||
:block/parent parent-id
|
||||
:block/left [:block/uuid (:block/uuid left)]
|
||||
:block/level (:block/level left)
|
||||
:block/level-spaces (:block/level-spaces left)))
|
||||
|
||||
parents' (->> (concat f [block]) vec)
|
||||
result' (conj result block)]
|
||||
[others parents' result'])))]
|
||||
(recur blocks parents result)))))
|
||||
|
||||
(defn parse-block
|
||||
([block]
|
||||
(parse-block block nil))
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
[frontend.db :as db]
|
||||
[frontend.db.model :as model]
|
||||
[frontend.handler.editor :as editor]
|
||||
[frontend.handler.extract :as extract]
|
||||
[logseq.graph-parser.extract :as extract]
|
||||
[frontend.handler.file :as file-handler]
|
||||
[frontend.handler.page :as page-handler]
|
||||
[frontend.handler.repo :as repo-handler]
|
||||
|
|
|
@ -1960,7 +1960,7 @@
|
|||
(let [blocks (block-tree->blocks tree-vec format)
|
||||
target-block (db/pull target-block-id)
|
||||
page-id (:db/id (:block/page target-block))
|
||||
blocks (block/with-parent-and-left page-id blocks)]
|
||||
blocks (gp-block/with-parent-and-left page-id blocks)]
|
||||
(paste-blocks
|
||||
blocks
|
||||
{:target-block target-block
|
||||
|
@ -2857,7 +2857,7 @@
|
|||
(let [page-id (:db/id (:block/page editing-block))
|
||||
blocks (block/extract-blocks
|
||||
(mldoc/->edn text (gp-mldoc/default-config format)) text true format)
|
||||
blocks' (block/with-parent-and-left page-id blocks)]
|
||||
blocks' (gp-block/with-parent-and-left page-id blocks)]
|
||||
(paste-blocks blocks' {}))))
|
||||
|
||||
(defn- paste-segmented-text
|
||||
|
|
|
@ -11,7 +11,7 @@
|
|||
[frontend.fs :as fs]
|
||||
[frontend.fs.nfs :as nfs]
|
||||
[frontend.handler.common :as common-handler]
|
||||
[frontend.handler.extract :as extract-handler]
|
||||
[logseq.graph-parser.extract :as extract]
|
||||
[frontend.handler.ui :as ui-handler]
|
||||
[frontend.state :as state]
|
||||
[frontend.util :as util]
|
||||
|
@ -123,11 +123,15 @@
|
|||
file-content [{:file/path file}]
|
||||
tx (if (contains? gp-config/mldoc-support-formats format)
|
||||
(let [[pages blocks]
|
||||
(extract-handler/extract-blocks-pages
|
||||
repo-url file content
|
||||
(extract/extract-blocks-pages
|
||||
file
|
||||
content
|
||||
{:user-config (state/get-config)
|
||||
:date-formatter (state/get-date-formatter)
|
||||
:page-name-order (state/page-name-order)})
|
||||
:page-name-order (state/page-name-order)
|
||||
:block-pattern (config/get-block-pattern format)
|
||||
:supported-formats (config/supported-formats)
|
||||
:db (db/get-db (state/get-current-repo))})
|
||||
first-page (first pages)
|
||||
delete-blocks (->
|
||||
(concat
|
||||
|
@ -149,7 +153,7 @@
|
|||
(seq))
|
||||
;; To prevent "unique constraint" on datascript
|
||||
block-ids (set/union (set block-ids) (set block-refs-ids))
|
||||
pages (extract-handler/with-ref-pages pages blocks)
|
||||
pages (extract/with-ref-pages pages blocks)
|
||||
pages-index (map #(select-keys % [:block/name]) pages)]
|
||||
;; does order matter?
|
||||
(concat file-content pages-index delete-blocks pages block-ids blocks))
|
||||
|
|
|
@ -581,3 +581,73 @@
|
|||
(map #(dissoc % :block/meta) result))
|
||||
(catch :default e
|
||||
(log/error :extract-blocks-failure e))))
|
||||
|
||||
(defn with-parent-and-left
|
||||
[page-id blocks]
|
||||
(loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
|
||||
parents [{:page/id page-id ; db id or a map {:block/name "xxx"}
|
||||
:block/level 0
|
||||
:block/level-spaces 0}]
|
||||
result []]
|
||||
(if (empty? blocks)
|
||||
(map #(dissoc % :block/level-spaces) result)
|
||||
(let [[block & others] blocks
|
||||
level-spaces (:block/level-spaces block)
|
||||
{:block/keys [uuid level parent] :as last-parent} (last parents)
|
||||
parent-spaces (:block/level-spaces last-parent)
|
||||
[blocks parents result]
|
||||
(cond
|
||||
(= level-spaces parent-spaces) ; sibling
|
||||
(let [block (assoc block
|
||||
:block/parent parent
|
||||
:block/left [:block/uuid uuid]
|
||||
:block/level level)
|
||||
parents' (conj (vec (butlast parents)) block)
|
||||
result' (conj result block)]
|
||||
[others parents' result'])
|
||||
|
||||
(> level-spaces parent-spaces) ; child
|
||||
(let [parent (if uuid [:block/uuid uuid] (:page/id last-parent))
|
||||
block (cond->
|
||||
(assoc block
|
||||
:block/parent parent
|
||||
:block/left parent)
|
||||
;; fix block levels with wrong order
|
||||
;; For example:
|
||||
;; - a
|
||||
;; - b
|
||||
;; What if the input indentation is two spaces instead of 4 spaces
|
||||
(>= (- level-spaces parent-spaces) 1)
|
||||
(assoc :block/level (inc level)))
|
||||
parents' (conj parents block)
|
||||
result' (conj result block)]
|
||||
[others parents' result'])
|
||||
|
||||
(< level-spaces parent-spaces)
|
||||
(cond
|
||||
(some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
|
||||
(let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
|
||||
left (last parents')
|
||||
blocks (cons (assoc (first blocks)
|
||||
:block/level (dec level)
|
||||
:block/left [:block/uuid (:block/uuid left)])
|
||||
(rest blocks))]
|
||||
[blocks parents' result])
|
||||
|
||||
:else
|
||||
(let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
|
||||
left (first r)
|
||||
parent-id (if-let [block-id (:block/uuid (last f))]
|
||||
[:block/uuid block-id]
|
||||
page-id)
|
||||
block (cond->
|
||||
(assoc block
|
||||
:block/parent parent-id
|
||||
:block/left [:block/uuid (:block/uuid left)]
|
||||
:block/level (:block/level left)
|
||||
:block/level-spaces (:block/level-spaces left)))
|
||||
|
||||
parents' (->> (concat f [block]) vec)
|
||||
result' (conj result block)]
|
||||
[others parents' result'])))]
|
||||
(recur blocks parents result)))))
|
||||
|
|
|
@ -1,17 +1,19 @@
|
|||
(ns frontend.handler.extract
|
||||
"Extract helper."
|
||||
(ns ^:nbb-compatible logseq.graph-parser.extract
|
||||
;; Disable clj linters since we don't support clj
|
||||
#?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
|
||||
:unresolved-symbol {:level :off}}}})
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[clojure.walk :as walk]
|
||||
[datascript.core :as d]
|
||||
[frontend.format.block :as block]
|
||||
[logseq.graph-parser.text :as text]
|
||||
[logseq.graph-parser.util :as gp-util]
|
||||
[logseq.graph-parser.mldoc :as gp-mldoc]
|
||||
[logseq.graph-parser.block :as gp-block]
|
||||
[logseq.graph-parser.property :as gp-property]
|
||||
[logseq.graph-parser.config :as gp-config]
|
||||
[lambdaisland.glogi :as log]))
|
||||
#?(:org.babashka/nbb [logseq.graph-parser.log :as log]
|
||||
:default [lambdaisland.glogi :as log])))
|
||||
|
||||
(defn- get-page-name
|
||||
[file ast page-name-order]
|
||||
|
@ -40,13 +42,13 @@
|
|||
|
||||
;; TODO: performance improvement
|
||||
(defn- extract-pages-and-blocks
|
||||
#_:clj-kondo/ignore
|
||||
[repo-url format ast properties file content date-formatter page-name-order]
|
||||
[format ast properties file content {:keys [date-formatter page-name-order db] :as options}]
|
||||
(try
|
||||
#_:clj-kondo/ignore ;;clj-kondo bug
|
||||
(let [page (get-page-name file ast page-name-order)
|
||||
[_original-page-name page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter)
|
||||
blocks (->> (block/extract-blocks ast content false format)
|
||||
(block/with-parent-and-left {:block/name page-name}))
|
||||
blocks (->> (gp-block/extract-blocks ast content false format (dissoc options :page-name-order))
|
||||
(gp-block/with-parent-and-left {:block/name page-name}))
|
||||
ref-pages (atom #{})
|
||||
ref-tags (atom #{})
|
||||
blocks (map (fn [block]
|
||||
|
@ -91,7 +93,7 @@
|
|||
(cond->
|
||||
(gp-util/remove-nils
|
||||
(assoc
|
||||
(block/page-name->map page false)
|
||||
(gp-block/page-name->map page false db true date-formatter)
|
||||
:block/file {:file/path (gp-util/path-normalize file)}))
|
||||
(seq properties)
|
||||
(assoc :block/properties properties)
|
||||
|
@ -111,7 +113,7 @@
|
|||
(when (text/namespace-page? page)
|
||||
(->> (gp-util/split-namespace-pages page)
|
||||
(map (fn [page]
|
||||
(-> (block/page-name->map page true)
|
||||
(-> (gp-block/page-name->map page true db true date-formatter)
|
||||
(assoc :block/format format)))))))
|
||||
pages (->> (concat
|
||||
[page-entity]
|
||||
|
@ -131,11 +133,11 @@
|
|||
blocks (->> (remove nil? blocks)
|
||||
(map (fn [b] (dissoc b :block/title :block/body :block/level :block/children :block/meta :block/anchor))))]
|
||||
[pages blocks])
|
||||
(catch js/Error e
|
||||
(catch :default e
|
||||
(log/error :exception e))))
|
||||
|
||||
(defn extract-blocks-pages
|
||||
[repo-url file content {:keys [user-config date-formatter page-name-order]}]
|
||||
[file content {:keys [user-config] :as options}]
|
||||
(if (string/blank? content)
|
||||
[]
|
||||
(let [format (gp-util/get-format file)
|
||||
|
@ -161,11 +163,10 @@
|
|||
(string/replace (or v "") "\\" "")))
|
||||
properties)))]
|
||||
(extract-pages-and-blocks
|
||||
repo-url
|
||||
format ast properties
|
||||
file content date-formatter page-name-order)))))
|
||||
file content options)))))
|
||||
|
||||
(defn with-block-uuid
|
||||
(defn- with-block-uuid
|
||||
[pages]
|
||||
(->> (gp-util/distinct-by :block/name pages)
|
||||
(map (fn [page]
|
|
@ -8,7 +8,7 @@
|
|||
[frontend.db :as db]
|
||||
[frontend.db.model :as db-model]
|
||||
[clojure.walk :as walk]
|
||||
[frontend.format.block :as block]
|
||||
[logseq.graph-parser.block :as gp-block]
|
||||
[datascript.core :as d]
|
||||
[frontend.test.helper :as helper]))
|
||||
|
||||
|
@ -65,7 +65,7 @@
|
|||
|
||||
(defn- build-blocks
|
||||
[tree]
|
||||
(block/with-parent-and-left 1 (build-node-tree tree)))
|
||||
(gp-block/with-parent-and-left 1 (build-node-tree tree)))
|
||||
|
||||
(defn transact-tree!
|
||||
[tree]
|
||||
|
|
|
@ -1,17 +1,17 @@
|
|||
(ns frontend.handler.extract-test
|
||||
(ns logseq.graph-parser.extract-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[frontend.handler.extract :as extract]
|
||||
[frontend.util :as util]
|
||||
[logseq.graph-parser.extract :as extract]
|
||||
[clojure.pprint :as pprint]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- extract
|
||||
[text]
|
||||
(p/let [result (extract/extract-blocks-pages "repo" "a.md" text {})
|
||||
(p/let [result (extract/extract-blocks-pages "a.md" text {:block-pattern "-"})
|
||||
result (last result)
|
||||
lefts (map (juxt :block/parent :block/left) result)]
|
||||
(if (not= (count lefts) (count (distinct lefts)))
|
||||
(do
|
||||
(util/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
|
||||
(pprint/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
|
||||
(throw (js/Error. ":block/parent && :block/left conflicts")))
|
||||
(mapv :block/content result))))
|
||||
|
|
@ -6,7 +6,8 @@
|
|||
[logseq.graph-parser.text-test]
|
||||
[logseq.graph-parser.mldoc-test]
|
||||
[logseq.graph-parser.block-test]
|
||||
[logseq.graph-parser.property-test]))
|
||||
[logseq.graph-parser.property-test]
|
||||
[logseq.graph-parser.extract-test]))
|
||||
|
||||
(defmethod cljs.test/report [:cljs.test/default :end-run-tests] [m]
|
||||
(when-not (cljs.test/successful? m)
|
||||
|
@ -21,4 +22,5 @@
|
|||
(t/run-tests 'logseq.graph-parser.mldoc-test
|
||||
'logseq.graph-parser.text-test
|
||||
'logseq.graph-parser.property-test
|
||||
'logseq.graph-parser.block-test))
|
||||
'logseq.graph-parser.block-test
|
||||
'logseq.graph-parser.extract-test))
|
||||
|
|
Loading…
Reference in New Issue