greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

chore: remove :block/title, :block/body and :block/unordered 

For two reasons:

1. Reducing memory usage, both block/title and block/body are AST parsed
by Mldoc. After removing, logseq is able to handle 10k notes.

2. Simplify the db schema
pull/3378/head
Tienson Qin 2021-12-06 19:20:56 +08:00
parent 1ca2b3fc21
commit 7822848af8
18 changed files with 92 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/main/frontend/components/block.cljs
View File

@ -48,7 +48,7 @@
[frontend.template :as template]
[frontend.text :as text]
[frontend.ui :as ui]
[frontend.util :as util]
[frontend.util :as util :refer [profile]]
[frontend.util.clock :as clock]
[frontend.util.property :as property]
[frontend.util.drawer :as drawer]
@ -643,13 +643,10 @@
hl-type (get-in block [:block/properties :hl-type])
repo (state/get-current-repo)]
(if block
(let [title (let [title (:block/title block)
block-content (block-content (assoc config :block-ref? true)
block nil (:block/uuid block)
(:slide? config))
class (if (seq title) "block-ref" "block-ref-no-title")]
[:span {:class class}
block-content])
(let [title [:span {:class "block-ref"}
(block-content (assoc config :block-ref? true)
block nil (:block/uuid block)
(:slide? config))]
inner (if label
(->elem
:span.block-ref
@ -1369,7 +1366,7 @@
(rum/with-key (block-container config child)
(:block/uuid child)))))]))))
(defn block-content-empty?
(defn- block-content-empty?
[{:block/keys [properties title body]}]
(and
(or
@ -1813,8 +1810,10 @@
summary]]))))])
(rum/defc block-content < rum/reactive
[config {:block/keys [uuid title body content children properties scheduled deadline] :as block} edit-input-id block-id slide?]
(let [collapsed? (get properties :collapsed)
[config {:block/keys [uuid content children properties scheduled deadline format pre-block?] :as block} edit-input-id block-id slide?]
(let [{:block/keys [title body] :as block} (if (:block/title block) block
(merge block (block/parse-title-and-body format pre-block? content)))
collapsed? (get properties :collapsed)
block-ref? (:block-ref? config)
block-ref-with-title? (and block-ref? (seq title))
block-type (or (:ls-type properties) :default)
@ -1999,12 +1998,13 @@
[:page
(or page-original-name page-name)])
parents-props (doall
(for [{:block/keys [uuid title body name] :as block} parents]
(for [{:block/keys [uuid name content] :as block} parents]
(when-not name ; not page
[block
(if (seq title)
(->elem :span (map-inline config title))
(->elem :div (markup-elements-cp config body)))])))
(let [{:block/keys [title body]} (block/parse-title-and-body (:block/format block) (:block/pre-block? block) content)]
[block
(if (seq title)
(->elem :span (map-inline config title))
(->elem :div (markup-elements-cp config body)))]))))
breadcrumb (->> (into [] parents-props)
(concat [page-name-props] (when more? [:more]))
(filterv identity)
@ -2159,8 +2159,10 @@
(select-keys (second (:rum/args new-state)) compare-keys))
(not= (select-keys (first (:rum/args old-state)) config-compare-keys)
(select-keys (first (:rum/args new-state)) config-compare-keys)))))}
[state config {:block/keys [uuid body repo children pre-block? top? properties refs heading-level level type] :as block}]
(let [blocks-container-id (:blocks-container-id config)
[state config {:block/keys [uuid repo children pre-block? top? properties refs heading-level level type format content] :as block}]
(let [block (merge block (block/parse-title-and-body format pre-block? content))
body (:block/body block)
blocks-container-id (:blocks-container-id config)
config (update config :block merge block)
;; Each block might have multiple queries, but we store only the first query's result
config (if (nil? (:query-result config))
@ -2384,7 +2386,7 @@
clocks))]
[:div.overflow-x-scroll.sm:overflow-auto
(->elem
:table.m-0
:table.m-0
{:class "logbook-table"
:border 0
:style {:width "max-content"}

9
src/main/frontend/components/query_table.cljs
View File

@ -9,6 +9,7 @@
[frontend.util :as util]
[frontend.util.clock :as clock]
[frontend.util.property :as property]
[frontend.format.block :as block]
[medley.core :as medley]
[rum.core :as rum]))
@ -47,7 +48,8 @@
[result]
(let [ks [:block/properties :clock-time]
result (map (fn [b]
(assoc-in b ks (or (clock/clock-summary (:block/body b) false) 0)))
(let [b (block/parse-title-and-body b)]
(assoc-in b ks (or (clock/clock-summary (:block/body b) false) 0))))
result)]
(if (every? #(zero? (get-in % ks)) result)
(map #(medley/dissoc-in % ks) result)
@ -125,10 +127,11 @@
(:block/name item))]
:block ; block title
(let [title (:block/title item)]
(let [content (:block/content item)
{:block/keys [title]} (block/parse-title-and-body (:block/format item) (:block/pre-block? item) content)]
(if (seq title)
[:element (->elem :div (map-inline config title))]
[:string (:block/content item)]))
[:string content]))
:created-at
[:string (when-let [created-at (:block/created-at item)]

9
src/main/frontend/components/sidebar.cljs
View File

@ -65,15 +65,6 @@
[:a.more svg/arrow-down-v2]]]
[:div.bd child]])
;; TODO: enhance
(defn- pick-one-ast-page-ref
[block]
(when-let [title-ast (and block (:block/title block))]
(when-let [link-ref (and (= (ffirst title-ast) "Link")
(:url (second (first title-ast))))]
(when (= "Page_ref" (first link-ref))
(second link-ref)))))
(defn- delta-y
[e]
(let [rect (.. (.. e -target) getBoundingClientRect)]

3
src/main/frontend/db/model.cljs
View File

@ -27,7 +27,6 @@
:block/type
:block/left
:block/format
:block/title
:block/refs
:block/_refs
:block/path-refs
@ -36,7 +35,6 @@
:block/marker
:block/priority
:block/properties
:block/body
:block/pre-block?
:block/scheduled
:block/deadline
@ -45,7 +43,6 @@
:block/updated-at
:block/file
:block/parent
:block/unordered
:block/heading-level
{:block/page [:db/id :block/name :block/original-name :block/journal-day]}
{:block/_parent ...}])

11
src/main/frontend/db_schema.cljs
View File

@ -30,9 +30,6 @@
;; :markdown, :org
:block/format {}
;; mldoc parsed ast
:block/title {}
;; belongs to which page
:block/page {:db/valueType :db.type/ref
:db/index true}
@ -71,9 +68,6 @@
;; vector
:block/properties-order {}
;; parsed ast
:block/body {}
;; first block that's not a heading or unordered list
:block/pre-block? {}
@ -140,8 +134,6 @@
:block/level
:block/heading-level
:block/type
:block/title
:block/body
:block/properties
:block/created-at
:block/updated-at
@ -160,5 +152,4 @@
:block/content
:block/properties
:block/alias
:block/tags
:block/unordered})
:block/tags})

35
src/main/frontend/format/block.cljs
View File

@ -716,24 +716,29 @@
new-block
{:block/path-refs path-ref-pages})
(> (count blocks) 1)
(assoc :block/warning :multiple-blocks))]
(assoc :block/warning :multiple-blocks))
block (dissoc block :block/title :block/body :block/level)]
(if uuid (assoc block :block/uuid uuid) block)))))
(defn parse-title-and-body
[format pre-block? content]
(def content content)
(let [ast (format/to-edn content format nil)
content (if pre-block? content
(str (config/get-block-pattern format) " " (string/triml content)))
content (property/remove-properties format content)
ast (->> (format/to-edn content format nil)
(map first))
title (when (heading-block? (first ast))
(:title (second (first ast))))]
(cond->
{:block/body (vec (if title (rest ast) ast))}
title
(assoc :block/title title))))
([block]
(when (map? block)
(merge block
(parse-title-and-body (:block/format block)
(:block/pre-block? block)
(:block/content block)))))
([format pre-block? content]
(let [content (if pre-block? content
(str (config/get-block-pattern format) " " (string/triml content)))
content (property/remove-properties format content)
ast (->> (format/to-edn content format nil)
(map first))
title (when (heading-block? (first ast))
(:title (second (first ast))))]
(cond->
{:block/body (vec (if title (rest ast) ast))}
title
(assoc :block/title title)))))
(defn macro-subs
[macro-content arguments]

2
src/main/frontend/handler/block.cljs
View File

@ -68,7 +68,7 @@
(remove nil?)))))))
;; TODO: reduced version
(defn walk-block
(defn- walk-block
[block check? transform]
(let [result (atom nil)]
(walk/postwalk

38
src/main/frontend/handler/editor.cljs
View File

@ -347,8 +347,10 @@
value))
(defn wrap-parse-block
[{:block/keys [content format left page uuid level] :as block}]
(let [block (or (and (:db/id block) (db/pull (:db/id block))) block)
[{:block/keys [content format left page uuid level pre-block?] :as block}]
(let [block (merge
(or (and (:db/id block) (db/pull (:db/id block))) block)
(block/parse-title-and-body format pre-block? content))
properties (:block/properties block)
real-content (:block/content block)
content (if (and (seq properties) real-content (not= real-content content))
@ -747,7 +749,6 @@
:block/format format
:block/content content
:block/parent page
:block/unordered true
:block/page page}))
(defn default-properties-block
@ -767,7 +768,6 @@
:block/format format
:block/content content
:block/parent page
:block/unordered true
:block/page page})))
(defn add-default-title-property-if-needed!
@ -2151,22 +2151,10 @@
[uuid page exclude-properties format content-update-fn]
(fn [block]
(outliner-core/block
(let [[new-content new-title]
(let [new-content
(if content-update-fn
(let [new-content (content-update-fn (:block/content block))
new-title (or (->> (mldoc/->edn
(str (case format
:markdown "- "
:org "* ")
(if (seq (:block/title block)) "" "\n")
new-content)
(mldoc/default-config format))
(ffirst)
(second)
(:title))
(:block/title block))]
[new-content new-title])
[(:block/content block) (:block/title block)])
(content-update-fn (:block/content block))
(:block/content block))
new-content
(->> new-content
(property/remove-property format "id")
@ -2185,7 +2173,6 @@
exclude-properties))
:block/meta (dissoc (:block/meta block) :start-pos :end-pos)
:block/content new-content
:block/title new-title
:block/path-refs (->> (cons (:db/id page) (:block/path-refs block))
(remove nil?))})]
m))))
@ -2441,7 +2428,7 @@
(when (thingatpt/get-setting :properties?)
(thingatpt/properties-at-point input))
(when (thingatpt/get-setting :list?)
(and (cursor/end-of-line? input) ;; only apply DWIM when cursor at EOL
(and (cursor/end-of-line? input) ;; only apply DWIM when cursor at EOL
(thingatpt/list-item-at-point input))))]
(cond
thing-at-point
@ -3313,10 +3300,11 @@
(defn collapsable? [block-id]
(if-let [block (db-model/get-block-by-uuid block-id)]
(and
(nil? (-> block :block/properties :collapsed))
(or (not-empty (:block/body block))
(db-model/has-children? block-id)))
(let [block (block/parse-title-and-body block)]
(and
(nil? (-> block :block/properties :collapsed))
(or (not-empty (:block/body block))
(db-model/has-children? block-id))))
false))
(defn collapse-block! [block-id]

2
src/main/frontend/handler/export.cljs
View File

@ -547,8 +547,6 @@
:block/heading-level
:block/format
:block/children
:block/title
:block/body
:block/content]))})
(defn- file-name [repo extension]

14
src/main/frontend/handler/extract.cljs
View File

@ -137,9 +137,10 @@
(remove nil? blocks))
(remove nil?))
pages (remove nil? pages)
pages (map (fn [page] (assoc page :block/uuid (db/new-block-id))) pages)]
[pages
(remove nil? blocks)])
pages (map (fn [page] (assoc page :block/uuid (db/new-block-id))) pages)
blocks (->> (remove nil? blocks)
(map (fn [b] (dissoc b :block/title :block/body :block/level))))]
[pages blocks])
(catch js/Error e
(log/error :exception e))))
@ -232,8 +233,11 @@
pages (with-ref-pages pages blocks)
blocks (map (fn [block]
(let [id (:block/uuid block)
properties (get-in metadata [:block/properties id])]
(update block :block/properties merge properties)))
properties (merge (get-in metadata [:block/properties id])
(:block/properties block))]
(if (seq properties)
(assoc block :block/properties properties)
(dissoc block :block/properties))))
blocks)
;; To prevent "unique constraint" on datascript
pages-index (map #(select-keys % [:block/name]) pages)

41
src/main/frontend/handler/page.cljs
View File

@ -126,31 +126,6 @@
(route-handler/redirect-to-page! page))
page)))))
(defn get-plugins
[blocks]
(let [plugins (atom {})
add-plugin #(swap! plugins assoc % true)]
(walk/postwalk
(fn [x]
(if (and (vector? x)
(>= (count x) 2))
(let [[type option] x]
(case type
"Src" (when (:language option)
(add-plugin "highlight"))
"Export" (when (= option "latex")
(add-plugin "latex"))
"Latex_Fragment" (add-plugin "latex")
"Math" (add-plugin "latex")
"Latex_Environment" (add-plugin "latex")
nil)
x)
x))
(map :block/body blocks))
@plugins))
(defn delete-file!
[repo page-name]
(let [file (db/get-page-file page-name)
@ -350,22 +325,20 @@
page-ids (->> (map :block/page blocks)
(remove nil?)
(set))
tx (->> (map (fn [{:block/keys [uuid title content properties format pre-block?] :as block}]
tx (->> (map (fn [{:block/keys [uuid content properties format pre-block?] :as block}]
(let [content (let [content' (replace-old-page! content old-original-name new-name)]
(when-not (= content' content)
content'))
properties (let [properties' (walk-replace-old-page! properties old-original-name new-name)]
(when-not (= properties' properties)
properties'))]
(when (or title content properties)
(when (or content properties)
(util/remove-nils-non-nested
(merge
{:block/uuid uuid
:block/content content
:block/properties properties
:block/refs (rename-update-block-refs! (:block/refs block) (:db/id page) (:db/id to-page))
:block/path-refs (rename-update-block-refs! (:block/path-refs block) (:db/id page) (:db/id to-page))}
(block/parse-title-and-body format pre-block? content)))))) blocks)
{:block/uuid uuid
:block/content content
:block/properties properties
:block/refs (rename-update-block-refs! (:block/refs block) (:db/id page) (:db/id to-page))
:block/path-refs (rename-update-block-refs! (:block/path-refs block) (:db/id page) (:db/id to-page))})))) blocks)
(remove nil?))]
(db/transact! repo tx)
(doseq [page-id page-ids]

10
src/main/frontend/modules/file/core.cljs
View File

@ -6,8 +6,9 @@
[frontend.db :as db]
[frontend.db.utils :as db-utils]
[frontend.state :as state]
[frontend.util :as util]
[frontend.debug :as debug]))
[frontend.util :as util :refer [profile]]
[frontend.debug :as debug]
[frontend.format.block :as block]))
(defn- indented-block-content
[content spaces-tabs]
@ -23,8 +24,9 @@
(ffirst body))))
(defn transform-content
[{:block/keys [format pre-block? title content unordered body heading-level left page scheduled deadline parent] :as block} level {:keys [heading-to-list?]}]
(let [content (or content "")
[{:block/keys [format pre-block? unordered content heading-level left page scheduled deadline parent] :as block} level {:keys [heading-to-list?]}]
(let [{:block/keys [title body]} (block/parse-title-and-body format pre-block? content)
content (or content "")
heading-with-title? (seq title)
allowed-block-as-title? (allowed-block-as-title? title body)
first-block? (= left page)

12
src/main/frontend/modules/outliner/core.cljs
View File

@ -60,15 +60,6 @@
(outliner-state/get-by-parent-id repo [:block/uuid id])
(mapv block))))
(defn- update-block-unordered
[block]
(let [parent (:block/parent block)
page (:block/page block)
type (:block/type block)]
(if (and parent page type (= parent page) (= type :heading))
(assoc block :block/unordered false)
(assoc block :block/unordered true))))
(defn- block-with-timestamps
[block]
(let [updated-at (util/time-ms)
@ -155,8 +146,7 @@
(-save [this txs-state]
(assert (ds/outliner-txs-state? txs-state)
"db should be satisfied outliner-tx-state?")
(let [this (block (update-block-unordered (:data this)))
m (-> (:data this)
(let [m (-> (:data this)
(dissoc :block/children :block/meta :block/top? :block/bottom?)
(util/remove-nils))
m (if (state/enable-block-timestamps?) (block-with-timestamps m) m)

6
src/main/frontend/tools/html_export.cljs
View File

@ -5,7 +5,8 @@
[frontend.components.block :as block]
[frontend.db :as db]
[frontend.extensions.slide :as slide]
[medley.core :as medley]))
[medley.core :as medley]
[frontend.format.block :as block]))
;; Consider generate a db index so that search can still works
@ -16,7 +17,8 @@
(defn- build-block
[config block]
(let [body (:block/body block)
(let [block (block/parse-title-and-body block)
body (:block/body block)
block (block/build-block-title config block)]
[:div.block
block

2
src/main/frontend/util/drawer.cljs
View File

@ -76,7 +76,7 @@
lines (concat [title] scheduled deadline before
[(drawer-start typ)] middle [drawer-end] after)]
(string/join "\n" lines))
:else
content)]
(string/trimr result))

2
src/main/frontend/util/page_property.cljs
View File

@ -109,7 +109,6 @@
:block/left page-id
:block/parent page-id
:block/page page-id
:block/title []
:block/content (if org?
(str "#+" (string/upper-case (name key)) ": " value)
(str (name key) ":: " value))
@ -124,4 +123,3 @@
:data [block]})
(ui-handler/re-render-root!)))
(outliner-file/sync-to-file page-id))))

2
src/test/frontend/handler/block_test.cljs
View File

@ -5,5 +5,5 @@
(comment
(defn clip-block [x]
(map #(select-keys % [:block/parent :block/left :block/pre-block? :block/uuid :block/level
:block/title :db/id])
:db/id])
x)))

6
src/test/frontend/modules/outliner/ds_test.cljs
View File

@ -15,12 +15,10 @@
:block/refs (),
:block/anchor "level_2123123",
:block/repo "logseq_local_test_navtive_fs",
:block/body [],
:block/meta {:timestamps [], :properties [], :start-pos 0, :end-pos 15},
:block/format :markdown,
:block/level 1,
:block/tags [],
:block/title [["Plain" "level 2123123"]],
:block/refs-with-children (),
:block/content "level test",
:db/id 72,
@ -29,14 +27,12 @@
rt [[72 :block/uuid #uuid "606c1962-ad7f-424e-b120-0dc7fcb25415" 536870913 true]
[72 :block/anchor "level_2123123" 536870913 true]
[72 :block/repo "logseq_local_test_navtive_fs" 536870913 true]
[72 :block/body [] 536870913 true]
[72 :block/meta {:timestamps [], :properties [], :start-pos 0, :end-pos 15} 536870913 true]
[72 :block/format :markdown 536870913 true]
[72 :block/level 1 536870913 true]
[72 :block/title [["Plain" "level 2123123"]] 536870913 true]
[72 :block/refs-with-children () 536870913 true]
[72 :block/content "level test" 536870913 true]]]
(is (= rt (mapv vec (:tx-data db-report))))))
(comment
(run-test test-with-db-macro))
(run-test test-with-db-macro))