greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move recalculate-block-path-refs to outliner dep 

Moved dependent query fns also into the outliner dep. Removed
get-block-children-ids from model as there was only one use of it after
the refactor. Also setup testing for outliner and moved in a test for it
pull/10438/head
Gabriel Horner 2023-09-06 16:20:29 -04:00
parent 128e03452f
commit a93fe4d81b
14 changed files with 182 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
deps/outliner/nbb.edn vendored
View File

@ -1,4 +1,6 @@
{:paths ["src"]
:deps
{logseq/db
{:local/root "../db"}}}
{:local/root "../db"}
io.github.nextjournal/nbb-test-runner
{:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}}

11
deps/outliner/package.json vendored Normal file
View File

@ -0,0 +1,11 @@
{
"name": "@logseq/outliner",
"version": "1.0.0",
"private": true,
"devDependencies": {
"@logseq/nbb-logseq": "^1.2.173"
},
"scripts": {
"test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"
}
}

100
deps/outliner/src/logseq/outliner/pipeline.cljs vendored
View File

@ -2,7 +2,8 @@
"Core fns for use with frontend.modules.outliner.pipeline"
(:require [logseq.db.schema :as db-schema]
[datascript.core :as d]
[cognitect.transit :as t]))
[cognitect.transit :as t]
[clojure.set :as set]))
(defn filter-deleted-blocks
[datoms]
@ -47,4 +48,99 @@
b)))
(map (fn [b]
(let [uuid (or (:block/uuid b) (random-uuid))]
(assoc b :block/uuid uuid)))))))
(assoc b :block/uuid uuid)))))))
;; non recursive query
(defn get-block-parents
[db block-id {:keys [depth] :or {depth 100}}]
(loop [block-id block-id
parents (list)
d 1]
(if (> d depth)
parents
(if-let [parent (:block/parent (d/entity db [:block/uuid block-id]))]
(recur (:block/uuid parent) (conj parents parent) (inc d))
parents))))
(defn get-block-children-ids
[db block-uuid]
(when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
(let [seen (volatile! [])]
(loop [steps 100 ;check result every 100 steps
eids-to-expand [eid]]
(when (seq eids-to-expand)
(let [eids-to-expand*
(mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
(when (and (zero? steps)
(seq (set/intersection (set @seen) (set uuids-to-add))))
(throw (ex-info "bad outliner data, need to re-index to fix"
{:seen @seen :eids-to-expand eids-to-expand})))
(vswap! seen (partial apply conj) uuids-to-add)
(recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
@seen)))
;; TODO: it'll be great if we can calculate the :block/path-refs before any
;; outliner transaction, this way we can group together the real outliner tx
;; and the new path-refs changes, which makes both undo/redo and
;; react-query/refresh! easier.
;; TODO: also need to consider whiteboard transactions
;; Steps:
;; 1. For each changed block, new-refs = its page + :block/refs + parents :block/refs
;; 2. Its children' block/path-refs might need to be updated too.
(defn computer-block-path-refs
[{:keys [db-before db-after]} blocks*]
(let [blocks (remove :block/name blocks*)
*computed-ids (atom #{})]
(mapcat (fn [block]
(when (and (not (@*computed-ids (:block/uuid block))) ; not computed yet
(not (:block/name block)))
(let [parents (get-block-parents db-after (:block/uuid block) {})
parents-refs (->> (mapcat :block/path-refs parents)
(map :db/id))
old-refs (if db-before
(set (map :db/id (:block/path-refs (d/entity db-before (:db/id block)))))
#{})
new-refs (set (concat
(some-> (:db/id (:block/page block)) vector)
(map :db/id (:block/refs block))
parents-refs))
refs-changed? (not= old-refs new-refs)
children (get-block-children-ids db-after (:block/uuid block))
;; Builds map of children ids to their parent id and :block/refs ids
children-maps (into {}
(map (fn [id]
(let [entity (d/entity db-after [:block/uuid id])]
[(:db/id entity)
{:parent-id (get-in entity [:block/parent :db/id])
:block-ref-ids (map :db/id (:block/refs entity))}]))
children))
children-refs (map (fn [[id {:keys [block-ref-ids] :as child-map}]]
{:db/id id
;; Recalculate :block/path-refs as db contains stale data for this attribute
:block/path-refs
(set/union
;; Refs from top-level parent
new-refs
;; Refs from current block
block-ref-ids
;; Refs from parents in between top-level
;; parent and current block
(loop [parent-refs #{}
parent-id (:parent-id child-map)]
(if-let [parent (children-maps parent-id)]
(recur (into parent-refs (:block-ref-ids parent))
(:parent-id parent))
;; exits when top-level parent is reached
parent-refs)))})
children-maps)]
(swap! *computed-ids set/union (set (cons (:block/uuid block) children)))
(concat
(when (and (seq new-refs)
refs-changed?)
[{:db/id (:db/id block)
:block/path-refs new-refs}])
children-refs))))
blocks)))

29
deps/outliner/test/logseq/outliner/pipeline_test.cljs vendored Normal file
View File

@ -0,0 +1,29 @@
(ns logseq.outliner.pipeline-test
(:require [cljs.test :refer [deftest is]]
[logseq.db.schema :as db-schema]
[datascript.core :as d]
[logseq.outliner.pipeline :as outliner-pipeline]))
;;; datoms
;;; - 1 <----+
;;; - 2 |
;;; - 3 -+
(def broken-outliner-data-with-cycle
[{:db/id 1
:block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
:block/parent 3}
{:db/id 2
:block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
:block/parent 1}
{:db/id 3
:block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
:block/parent 2}])
(deftest get-block-children-ids-on-bad-outliner-data
(let [db (d/db-with (d/empty-db db-schema/schema)
broken-outliner-data-with-cycle)]
(is (= "bad outliner data, need to re-index to fix"
(try (outliner-pipeline/get-block-children-ids db #uuid "e538d319-48d4-4a6d-ae70-c03bb55b6fe4")
(catch :default e
(ex-message e)))))))

15
deps/outliner/yarn.lock vendored Normal file
View File

@ -0,0 +1,15 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
"@logseq/nbb-logseq@^1.2.173":
version "1.2.173"
resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.2.173.tgz#27a52c350f06ac9c337d73687738f6ea8b2fc3f3"
integrity sha512-ABKPtVnSOiS4Zpk9+UTaGcs5H6EUmRADr9FJ0aEAVpa0WfAyvUbX/NgkQGMe1kKRv3EbIuLwaxfy+txr31OtAg==
dependencies:
import-meta-resolve "^2.1.0"
import-meta-resolve@^2.1.0:
version "2.2.2"
resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-2.2.2.tgz#75237301e72d1f0fbd74dbc6cca9324b164c2cc9"
integrity sha512-f8KcQ1D80V7RnqVm+/lirO9zkOxjGxhaTC1IPrBGd3MEfNgmNG67tSUO9gTi2F3Blr2Az6g1vocaxzkVnWl9MA==

2
src/main/frontend/components/block.cljs
View File

@ -2590,7 +2590,7 @@
level-limit 3}
:as opts}]
(when block-id
(let [parents (db/get-block-parents repo block-id (inc level-limit))
(let [parents (db/get-block-parents repo block-id {:depth (inc level-limit)})
page (or (db/get-block-page repo block-id) ;; only return for block uuid
(model/query-block-by-uuid block-id)) ;; return page entity when received page uuid
page-name (:block/name page)

2
src/main/frontend/db.cljs
View File

@ -33,7 +33,7 @@
delete-files delete-pages-by-files get-all-block-contents get-all-tagged-pages
get-all-templates get-block-and-children get-block-by-uuid get-block-children sort-by-left
get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid
get-block-children-ids get-block-immediate-children get-block-page
get-block-immediate-children get-block-page
get-custom-css get-date-scheduled-or-deadlines
get-file-last-modified-at get-file get-file-page get-file-page-id file-exists?
get-files get-files-blocks get-files-full get-journals-length get-pages-with-file

47
src/main/frontend/db/model.cljs
View File

@ -20,6 +20,7 @@
[logseq.graph-parser.util.page-ref :as page-ref]
[logseq.graph-parser.util.db :as db-util]
[logseq.graph-parser.util :as gp-util]
[logseq.outliner.pipeline :as outliner-pipeline]
[cljs-time.core :as t]
[cljs-time.format :as tf]
;; add map ops to datascript Entity
@ -535,19 +536,10 @@ independent of format as format specific heading characters are stripped"
(when-let [block (db-utils/entity db [:block/uuid block-id])]
(:block/parent block)))))
;; non recursive query
(defn get-block-parents
([repo block-id]
(get-block-parents repo block-id 100))
([repo block-id depth]
(loop [block-id block-id
parents (list)
d 1]
(if (> d depth)
parents
(if-let [parent (get-block-parent repo block-id)]
(recur (:block/uuid parent) (conj parents parent) (inc d))
parents)))))
[repo block-id opts]
(when-let [db (conn/get-db repo)]
(outliner-pipeline/get-block-parents db block-id opts)))
;; Use built-in recursive
(defn get-block-parents-v2
@ -702,28 +694,6 @@ independent of format as format specific heading characters are stripped"
(db-utils/pull-many repo
'[:db/id :block/name :block/original-name]
ids))))))
(defn get-block-children-ids-in-db
[db block-uuid]
(when-let [eid (:db/id (db-utils/entity db [:block/uuid block-uuid]))]
(let [seen (volatile! [])]
(loop [steps 100 ;check result every 100 steps
eids-to-expand [eid]]
(when (seq eids-to-expand)
(let [eids-to-expand*
(mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
uuids-to-add (remove nil? (map #(:block/uuid (db-utils/entity db %)) eids-to-expand*))]
(when (and (zero? steps)
(seq (set/intersection (set @seen) (set uuids-to-add))))
(throw (ex-info "bad outliner data, need to re-index to fix"
{:seen @seen :eids-to-expand eids-to-expand})))
(vswap! seen (partial apply conj) uuids-to-add)
(recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
@seen)))
(defn get-block-children-ids
([repo block-uuid]
(when-let [db (conn/get-db repo)]
(get-block-children-ids-in-db db block-uuid))))
(defn get-block-immediate-children
"Doesn't include nested children."
@ -735,10 +705,11 @@ independent of format as format specific heading characters are stripped"
(defn get-block-children
"Including nested children."
[repo block-uuid]
(let [ids (get-block-children-ids repo block-uuid)]
(when (seq ids)
(let [ids' (map (fn [id] [:block/uuid id]) ids)]
(db-utils/pull-many repo '[*] ids')))))
(when-let [db (conn/get-db repo)]
(let [ids (outliner-pipeline/get-block-children-ids db block-uuid)]
(when (seq ids)
(let [ids' (map (fn [id] [:block/uuid id]) ids)]
(db-utils/pull-many repo '[*] ids'))))))
;; TODO: use the tree directly
(defn- flatten-tree

2
src/main/frontend/handler/editor.cljs
View File

@ -1600,7 +1600,7 @@
(let [current-block (state/get-edit-block)
block-parents (set (->> (db/get-block-parents (state/get-current-repo)
block-id
99)
{:depth 99})
(map (comp str :block/uuid))))
current-and-parents (set/union #{(str (:block/uuid current-block))} block-parents)]
(p/let [result (search/block-search (state/get-current-repo) q {:limit 20})]

13
src/main/frontend/modules/outliner/core.cljs
View File

@ -18,10 +18,10 @@
[frontend.format.block :as block]
[frontend.handler.file-based.property.util :as property-util]
[frontend.handler.property.util :as pu]
[frontend.db.rtc.op :as rtc-op]
[frontend.format.mldoc :as mldoc]
[dommy.core :as dom]
[goog.object :as gobj]))
[goog.object :as gobj]
[logseq.outliner.pipeline :as outliner-pipeline]))
(s/def ::block-map (s/keys :opt [:db/id :block/uuid :block/page :block/left :block/parent]))
@ -858,7 +858,7 @@
(db/get-block-parents
(state/get-current-repo)
(tree/-get-id end-node)
1000)
{:depth 1000})
(map :block/uuid)
(set))
self-block? (contains? end-node-parents (tree/-get-id start-node))]
@ -878,7 +878,7 @@
(db/get-block-parents
(state/get-current-repo)
(tree/-get-id start-node)
1000)
{:depth 1000})
(map :block/uuid)
(set))
result (first (set/intersection (set end-node-left-nodes) parents))]
@ -917,7 +917,7 @@
original-position? (move-to-original-position? blocks target-block sibling? non-consecutive-blocks?)]
(when (and (not (contains? (set (map :db/id blocks)) (:db/id target-block)))
(not original-position?))
(let [parents (->> (db/get-block-parents (state/get-current-repo) (:block/uuid target-block))
(let [parents (->> (db/get-block-parents (state/get-current-repo) (:block/uuid target-block) {})
(map :db/id)
(set))
move-parents-to-child? (some parents (map :db/id blocks))]
@ -933,7 +933,8 @@
move-blocks-next-tx [(build-move-blocks-next-tx target-block blocks {:sibling? sibling?
:non-consecutive-blocks? non-consecutive-blocks?})]
children-page-tx (when not-same-page?
(let [children-ids (mapcat #(db/get-block-children-ids (state/get-current-repo) (:block/uuid %)) blocks)]
(let [children-ids (mapcat #(outliner-pipeline/get-block-children-ids (db/get-db (state/get-current-repo)) (:block/uuid %))
blocks)]
(map (fn [id] {:block/uuid id
:block/page target-page}) children-ids)))
fix-non-consecutive-tx (->> (fix-non-consecutive-blocks blocks target-block sibling?)

74
src/main/frontend/modules/outliner/pipeline.cljs
View File

@ -1,9 +1,6 @@
(ns frontend.modules.outliner.pipeline
(:require [clojure.set :as set]
[datascript.core :as d]
[frontend.config :as config]
(:require [frontend.config :as config]
[frontend.db :as db]
[frontend.db.model :as db-model]
[frontend.db.react :as react]
[frontend.modules.outliner.file :as file]
[logseq.outliner.datascript-report :as ds-report]
@ -20,73 +17,10 @@
(not (get-in tx-report [:tx-meta :created-from-journal-template?])))
(file/sync-to-file page (:outliner-op (:tx-meta tx-report)))))
;; TODO: it'll be great if we can calculate the :block/path-refs before any
;; outliner transaction, this way we can group together the real outliner tx
;; and the new path-refs changes, which makes both undo/redo and
;; react-query/refresh! easier.
;; TODO: also need to consider whiteboard transactions
;; Steps:
;; 1. For each changed block, new-refs = its page + :block/refs + parents :block/refs
;; 2. Its children' block/path-refs might need to be updated too.
(defn compute-block-path-refs
[{:keys [tx-meta db-before]} blocks]
(let [repo (state/get-current-repo)
blocks (remove :block/name blocks)]
(when (:outliner-op tx-meta)
(when (react/path-refs-need-recalculated? tx-meta)
(let [*computed-ids (atom #{})]
(mapcat (fn [block]
(when (and (not (@*computed-ids (:block/uuid block))) ; not computed yet
(not (:block/name block)))
(let [parents (db-model/get-block-parents repo (:block/uuid block))
parents-refs (->> (mapcat :block/path-refs parents)
(map :db/id))
old-refs (if db-before
(set (map :db/id (:block/path-refs (d/entity db-before (:db/id block)))))
#{})
new-refs (set (util/concat-without-nil
[(:db/id (:block/page block))]
(map :db/id (:block/refs block))
parents-refs))
refs-changed? (not= old-refs new-refs)
children (db-model/get-block-children-ids repo (:block/uuid block))
;; Builds map of children ids to their parent id and :block/refs ids
children-maps (into {}
(map (fn [id]
(let [entity (db/entity [:block/uuid id])]
[(:db/id entity)
{:parent-id (get-in entity [:block/parent :db/id])
:block-ref-ids (map :db/id (:block/refs entity))}]))
children))
children-refs (map (fn [[id {:keys [block-ref-ids] :as child-map}]]
{:db/id id
;; Recalculate :block/path-refs as db contains stale data for this attribute
:block/path-refs
(set/union
;; Refs from top-level parent
new-refs
;; Refs from current block
block-ref-ids
;; Refs from parents in between top-level
;; parent and current block
(loop [parent-refs #{}
parent-id (:parent-id child-map)]
(if-let [parent (children-maps parent-id)]
(recur (into parent-refs (:block-ref-ids parent))
(:parent-id parent))
;; exits when top-level parent is reached
parent-refs)))})
children-maps)]
(swap! *computed-ids set/union (set (cons (:block/uuid block) children)))
(util/concat-without-nil
[(when (and (seq new-refs)
refs-changed?)
{:db/id (:db/id block)
:block/path-refs new-refs})]
children-refs))))
blocks))))))
[{:keys [tx-meta] :as tx-report} blocks]
(when (and (:outliner-op tx-meta) (react/path-refs-need-recalculated? tx-meta))
(outliner-pipeline/computer-block-path-refs tx-report blocks)))
(defn invoke-hooks
[tx-report]

14
src/test/fixtures/broken-outliner-data-with-cycle.edn vendored
View File

@ -1,14 +0,0 @@
;;; datoms
;;; - 1 <----+
;;; - 2 |
;;; - 3 -+
[{:db/id 1
:block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
:block/parent 3}
{:db/id 2
:block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
:block/parent 1}
{:db/id 3
:block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
:block/parent 2}
]

17
src/test/frontend/db/model_test.cljs
View File

@ -3,12 +3,9 @@
[frontend.db.model :as model]
[frontend.db :as db]
[frontend.db.conn :as conn]
[logseq.db.schema :as db-schema]
[frontend.test.helper :as test-helper :refer [load-test-files]]
[datascript.core :as d]
[shadow.resource :as rc]
[clojure.set :as set]
[clojure.edn :as edn]))
[clojure.set :as set]))
(use-fixtures :each {:before test-helper/start-test-db!
:after test-helper/destroy-test-db!})
@ -151,18 +148,6 @@ foo:: bar"}])
"Non header block's content returns nil"))
(def broken-outliner-data-with-cycle (-> (rc/inline "fixtures/broken-outliner-data-with-cycle.edn")
edn/read-string))
(deftest get-block-children-ids-on-bad-outliner-data
(let [db (d/db-with (d/empty-db db-schema/schema)
broken-outliner-data-with-cycle)]
(is (= "bad outliner data, need to re-index to fix"
(try (model/get-block-children-ids-in-db db #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4")
(catch :default e
(ex-message e)))))))
(deftest get-block-immediate-children
(load-test-files [{:file/path "pages/page1.md"
:file/content "\n

3
src/test/frontend/modules/outliner/pipeline_test.cljs
View File

@ -14,6 +14,7 @@
db)
(map first)))
;; TODO: Move this test to outliner dep when there is a load-test-files helper for deps
(deftest compute-block-path-refs
(load-test-files [{:file/path "pages/page1.md"
:file/content "prop:: #bar
@ -31,7 +32,7 @@
:block/path-refs [{:db/id new-tag-id}])
%)
blocks)
refs-tx (pipeline/compute-block-path-refs {:tx-meta {:outliner-op :save-block}} modified-blocks)
refs-tx (pipeline/compute-block-path-refs {:tx-meta {:outliner-op :save-block} :db-after @conn} modified-blocks)
_ (d/transact! conn (concat (map (fn [m] [:db/retract (:db/id m) :block/path-refs]) refs-tx)
refs-tx))
updated-blocks (->> (get-blocks @conn)