diff --git a/deps/outliner/nbb.edn b/deps/outliner/nbb.edn index 01c02951b..90583c25b 100644 --- a/deps/outliner/nbb.edn +++ b/deps/outliner/nbb.edn @@ -1,4 +1,6 @@ {:paths ["src"] :deps {logseq/db - {:local/root "../db"}}} + {:local/root "../db"} + io.github.nextjournal/nbb-test-runner + {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}} diff --git a/deps/outliner/package.json b/deps/outliner/package.json new file mode 100644 index 000000000..efbb290ca --- /dev/null +++ b/deps/outliner/package.json @@ -0,0 +1,11 @@ +{ + "name": "@logseq/outliner", + "version": "1.0.0", + "private": true, + "devDependencies": { + "@logseq/nbb-logseq": "^1.2.173" + }, + "scripts": { + "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner" + } +} diff --git a/deps/outliner/src/logseq/outliner/pipeline.cljs b/deps/outliner/src/logseq/outliner/pipeline.cljs index 62f8fb66d..15286865e 100644 --- a/deps/outliner/src/logseq/outliner/pipeline.cljs +++ b/deps/outliner/src/logseq/outliner/pipeline.cljs @@ -2,7 +2,8 @@ "Core fns for use with frontend.modules.outliner.pipeline" (:require [logseq.db.schema :as db-schema] [datascript.core :as d] - [cognitect.transit :as t])) + [cognitect.transit :as t] + [clojure.set :as set])) (defn filter-deleted-blocks [datoms] @@ -47,4 +48,99 @@ b))) (map (fn [b] (let [uuid (or (:block/uuid b) (random-uuid))] - (assoc b :block/uuid uuid))))))) \ No newline at end of file + (assoc b :block/uuid uuid))))))) + +;; non recursive query +(defn get-block-parents + [db block-id {:keys [depth] :or {depth 100}}] + (loop [block-id block-id + parents (list) + d 1] + (if (> d depth) + parents + (if-let [parent (:block/parent (d/entity db [:block/uuid block-id]))] + (recur (:block/uuid parent) (conj parents parent) (inc d)) + parents)))) + +(defn get-block-children-ids + [db block-uuid] + (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))] + (let [seen (volatile! [])] + (loop [steps 100 ;check result every 100 steps + eids-to-expand [eid]] + (when (seq eids-to-expand) + (let [eids-to-expand* + (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand) + uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))] + (when (and (zero? steps) + (seq (set/intersection (set @seen) (set uuids-to-add)))) + (throw (ex-info "bad outliner data, need to re-index to fix" + {:seen @seen :eids-to-expand eids-to-expand}))) + (vswap! seen (partial apply conj) uuids-to-add) + (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*)))) + @seen))) + +;; TODO: it'll be great if we can calculate the :block/path-refs before any +;; outliner transaction, this way we can group together the real outliner tx +;; and the new path-refs changes, which makes both undo/redo and +;; react-query/refresh! easier. + +;; TODO: also need to consider whiteboard transactions + +;; Steps: +;; 1. For each changed block, new-refs = its page + :block/refs + parents :block/refs +;; 2. Its children' block/path-refs might need to be updated too. +(defn computer-block-path-refs + [{:keys [db-before db-after]} blocks*] + (let [blocks (remove :block/name blocks*) + *computed-ids (atom #{})] + (mapcat (fn [block] + (when (and (not (@*computed-ids (:block/uuid block))) ; not computed yet + (not (:block/name block))) + (let [parents (get-block-parents db-after (:block/uuid block) {}) + parents-refs (->> (mapcat :block/path-refs parents) + (map :db/id)) + old-refs (if db-before + (set (map :db/id (:block/path-refs (d/entity db-before (:db/id block))))) + #{}) + new-refs (set (concat + (some-> (:db/id (:block/page block)) vector) + (map :db/id (:block/refs block)) + parents-refs)) + refs-changed? (not= old-refs new-refs) + children (get-block-children-ids db-after (:block/uuid block)) + ;; Builds map of children ids to their parent id and :block/refs ids + children-maps (into {} + (map (fn [id] + (let [entity (d/entity db-after [:block/uuid id])] + [(:db/id entity) + {:parent-id (get-in entity [:block/parent :db/id]) + :block-ref-ids (map :db/id (:block/refs entity))}])) + children)) + children-refs (map (fn [[id {:keys [block-ref-ids] :as child-map}]] + {:db/id id + ;; Recalculate :block/path-refs as db contains stale data for this attribute + :block/path-refs + (set/union + ;; Refs from top-level parent + new-refs + ;; Refs from current block + block-ref-ids + ;; Refs from parents in between top-level + ;; parent and current block + (loop [parent-refs #{} + parent-id (:parent-id child-map)] + (if-let [parent (children-maps parent-id)] + (recur (into parent-refs (:block-ref-ids parent)) + (:parent-id parent)) + ;; exits when top-level parent is reached + parent-refs)))}) + children-maps)] + (swap! *computed-ids set/union (set (cons (:block/uuid block) children))) + (concat + (when (and (seq new-refs) + refs-changed?) + [{:db/id (:db/id block) + :block/path-refs new-refs}]) + children-refs)))) + blocks))) diff --git a/deps/outliner/test/logseq/outliner/pipeline_test.cljs b/deps/outliner/test/logseq/outliner/pipeline_test.cljs new file mode 100644 index 000000000..7f39db4f0 --- /dev/null +++ b/deps/outliner/test/logseq/outliner/pipeline_test.cljs @@ -0,0 +1,29 @@ +(ns logseq.outliner.pipeline-test + (:require [cljs.test :refer [deftest is]] + [logseq.db.schema :as db-schema] + [datascript.core :as d] + [logseq.outliner.pipeline :as outliner-pipeline])) + + +;;; datoms +;;; - 1 <----+ +;;; - 2 | +;;; - 3 -+ +(def broken-outliner-data-with-cycle + [{:db/id 1 + :block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4" + :block/parent 3} + {:db/id 2 + :block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5" + :block/parent 1} + {:db/id 3 + :block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2" + :block/parent 2}]) + +(deftest get-block-children-ids-on-bad-outliner-data + (let [db (d/db-with (d/empty-db db-schema/schema) + broken-outliner-data-with-cycle)] + (is (= "bad outliner data, need to re-index to fix" + (try (outliner-pipeline/get-block-children-ids db #uuid "e538d319-48d4-4a6d-ae70-c03bb55b6fe4") + (catch :default e + (ex-message e))))))) \ No newline at end of file diff --git a/deps/outliner/yarn.lock b/deps/outliner/yarn.lock new file mode 100644 index 000000000..66b8e38c2 --- /dev/null +++ b/deps/outliner/yarn.lock @@ -0,0 +1,15 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@logseq/nbb-logseq@^1.2.173": + version "1.2.173" + resolved "https://registry.yarnpkg.com/@logseq/nbb-logseq/-/nbb-logseq-1.2.173.tgz#27a52c350f06ac9c337d73687738f6ea8b2fc3f3" + integrity sha512-ABKPtVnSOiS4Zpk9+UTaGcs5H6EUmRADr9FJ0aEAVpa0WfAyvUbX/NgkQGMe1kKRv3EbIuLwaxfy+txr31OtAg== + dependencies: + import-meta-resolve "^2.1.0" + +import-meta-resolve@^2.1.0: + version "2.2.2" + resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-2.2.2.tgz#75237301e72d1f0fbd74dbc6cca9324b164c2cc9" + integrity sha512-f8KcQ1D80V7RnqVm+/lirO9zkOxjGxhaTC1IPrBGd3MEfNgmNG67tSUO9gTi2F3Blr2Az6g1vocaxzkVnWl9MA== diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index f85903937..fed816b10 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -2590,7 +2590,7 @@ level-limit 3} :as opts}] (when block-id - (let [parents (db/get-block-parents repo block-id (inc level-limit)) + (let [parents (db/get-block-parents repo block-id {:depth (inc level-limit)}) page (or (db/get-block-page repo block-id) ;; only return for block uuid (model/query-block-by-uuid block-id)) ;; return page entity when received page uuid page-name (:block/name page) diff --git a/src/main/frontend/db.cljs b/src/main/frontend/db.cljs index c94e8b910..6a4bae3c4 100644 --- a/src/main/frontend/db.cljs +++ b/src/main/frontend/db.cljs @@ -33,7 +33,7 @@ delete-files delete-pages-by-files get-all-block-contents get-all-tagged-pages get-all-templates get-block-and-children get-block-by-uuid get-block-children sort-by-left get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid - get-block-children-ids get-block-immediate-children get-block-page + get-block-immediate-children get-block-page get-custom-css get-date-scheduled-or-deadlines get-file-last-modified-at get-file get-file-page get-file-page-id file-exists? get-files get-files-blocks get-files-full get-journals-length get-pages-with-file diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 993c1f5bc..f0a660bf7 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -20,6 +20,7 @@ [logseq.graph-parser.util.page-ref :as page-ref] [logseq.graph-parser.util.db :as db-util] [logseq.graph-parser.util :as gp-util] + [logseq.outliner.pipeline :as outliner-pipeline] [cljs-time.core :as t] [cljs-time.format :as tf] ;; add map ops to datascript Entity @@ -535,19 +536,10 @@ independent of format as format specific heading characters are stripped" (when-let [block (db-utils/entity db [:block/uuid block-id])] (:block/parent block))))) -;; non recursive query (defn get-block-parents - ([repo block-id] - (get-block-parents repo block-id 100)) - ([repo block-id depth] - (loop [block-id block-id - parents (list) - d 1] - (if (> d depth) - parents - (if-let [parent (get-block-parent repo block-id)] - (recur (:block/uuid parent) (conj parents parent) (inc d)) - parents))))) + [repo block-id opts] + (when-let [db (conn/get-db repo)] + (outliner-pipeline/get-block-parents db block-id opts))) ;; Use built-in recursive (defn get-block-parents-v2 @@ -702,28 +694,6 @@ independent of format as format specific heading characters are stripped" (db-utils/pull-many repo '[:db/id :block/name :block/original-name] ids)))))) -(defn get-block-children-ids-in-db - [db block-uuid] - (when-let [eid (:db/id (db-utils/entity db [:block/uuid block-uuid]))] - (let [seen (volatile! [])] - (loop [steps 100 ;check result every 100 steps - eids-to-expand [eid]] - (when (seq eids-to-expand) - (let [eids-to-expand* - (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand) - uuids-to-add (remove nil? (map #(:block/uuid (db-utils/entity db %)) eids-to-expand*))] - (when (and (zero? steps) - (seq (set/intersection (set @seen) (set uuids-to-add)))) - (throw (ex-info "bad outliner data, need to re-index to fix" - {:seen @seen :eids-to-expand eids-to-expand}))) - (vswap! seen (partial apply conj) uuids-to-add) - (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*)))) - @seen))) - -(defn get-block-children-ids - ([repo block-uuid] - (when-let [db (conn/get-db repo)] - (get-block-children-ids-in-db db block-uuid)))) (defn get-block-immediate-children "Doesn't include nested children." @@ -735,10 +705,11 @@ independent of format as format specific heading characters are stripped" (defn get-block-children "Including nested children." [repo block-uuid] - (let [ids (get-block-children-ids repo block-uuid)] - (when (seq ids) - (let [ids' (map (fn [id] [:block/uuid id]) ids)] - (db-utils/pull-many repo '[*] ids'))))) + (when-let [db (conn/get-db repo)] + (let [ids (outliner-pipeline/get-block-children-ids db block-uuid)] + (when (seq ids) + (let [ids' (map (fn [id] [:block/uuid id]) ids)] + (db-utils/pull-many repo '[*] ids')))))) ;; TODO: use the tree directly (defn- flatten-tree diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 5061bc687..ad60665b6 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -1600,7 +1600,7 @@ (let [current-block (state/get-edit-block) block-parents (set (->> (db/get-block-parents (state/get-current-repo) block-id - 99) + {:depth 99}) (map (comp str :block/uuid)))) current-and-parents (set/union #{(str (:block/uuid current-block))} block-parents)] (p/let [result (search/block-search (state/get-current-repo) q {:limit 20})] diff --git a/src/main/frontend/modules/outliner/core.cljs b/src/main/frontend/modules/outliner/core.cljs index e387eab4c..58527f442 100644 --- a/src/main/frontend/modules/outliner/core.cljs +++ b/src/main/frontend/modules/outliner/core.cljs @@ -18,10 +18,10 @@ [frontend.format.block :as block] [frontend.handler.file-based.property.util :as property-util] [frontend.handler.property.util :as pu] - [frontend.db.rtc.op :as rtc-op] [frontend.format.mldoc :as mldoc] [dommy.core :as dom] - [goog.object :as gobj])) + [goog.object :as gobj] + [logseq.outliner.pipeline :as outliner-pipeline])) (s/def ::block-map (s/keys :opt [:db/id :block/uuid :block/page :block/left :block/parent])) @@ -858,7 +858,7 @@ (db/get-block-parents (state/get-current-repo) (tree/-get-id end-node) - 1000) + {:depth 1000}) (map :block/uuid) (set)) self-block? (contains? end-node-parents (tree/-get-id start-node))] @@ -878,7 +878,7 @@ (db/get-block-parents (state/get-current-repo) (tree/-get-id start-node) - 1000) + {:depth 1000}) (map :block/uuid) (set)) result (first (set/intersection (set end-node-left-nodes) parents))] @@ -917,7 +917,7 @@ original-position? (move-to-original-position? blocks target-block sibling? non-consecutive-blocks?)] (when (and (not (contains? (set (map :db/id blocks)) (:db/id target-block))) (not original-position?)) - (let [parents (->> (db/get-block-parents (state/get-current-repo) (:block/uuid target-block)) + (let [parents (->> (db/get-block-parents (state/get-current-repo) (:block/uuid target-block) {}) (map :db/id) (set)) move-parents-to-child? (some parents (map :db/id blocks))] @@ -933,7 +933,8 @@ move-blocks-next-tx [(build-move-blocks-next-tx target-block blocks {:sibling? sibling? :non-consecutive-blocks? non-consecutive-blocks?})] children-page-tx (when not-same-page? - (let [children-ids (mapcat #(db/get-block-children-ids (state/get-current-repo) (:block/uuid %)) blocks)] + (let [children-ids (mapcat #(outliner-pipeline/get-block-children-ids (db/get-db (state/get-current-repo)) (:block/uuid %)) + blocks)] (map (fn [id] {:block/uuid id :block/page target-page}) children-ids))) fix-non-consecutive-tx (->> (fix-non-consecutive-blocks blocks target-block sibling?) diff --git a/src/main/frontend/modules/outliner/pipeline.cljs b/src/main/frontend/modules/outliner/pipeline.cljs index b8bff8347..38eec0bd2 100644 --- a/src/main/frontend/modules/outliner/pipeline.cljs +++ b/src/main/frontend/modules/outliner/pipeline.cljs @@ -1,9 +1,6 @@ (ns frontend.modules.outliner.pipeline - (:require [clojure.set :as set] - [datascript.core :as d] - [frontend.config :as config] + (:require [frontend.config :as config] [frontend.db :as db] - [frontend.db.model :as db-model] [frontend.db.react :as react] [frontend.modules.outliner.file :as file] [logseq.outliner.datascript-report :as ds-report] @@ -20,73 +17,10 @@ (not (get-in tx-report [:tx-meta :created-from-journal-template?]))) (file/sync-to-file page (:outliner-op (:tx-meta tx-report))))) -;; TODO: it'll be great if we can calculate the :block/path-refs before any -;; outliner transaction, this way we can group together the real outliner tx -;; and the new path-refs changes, which makes both undo/redo and -;; react-query/refresh! easier. - -;; TODO: also need to consider whiteboard transactions - -;; Steps: -;; 1. For each changed block, new-refs = its page + :block/refs + parents :block/refs -;; 2. Its children' block/path-refs might need to be updated too. (defn compute-block-path-refs - [{:keys [tx-meta db-before]} blocks] - (let [repo (state/get-current-repo) - blocks (remove :block/name blocks)] - (when (:outliner-op tx-meta) - (when (react/path-refs-need-recalculated? tx-meta) - (let [*computed-ids (atom #{})] - (mapcat (fn [block] - (when (and (not (@*computed-ids (:block/uuid block))) ; not computed yet - (not (:block/name block))) - (let [parents (db-model/get-block-parents repo (:block/uuid block)) - parents-refs (->> (mapcat :block/path-refs parents) - (map :db/id)) - old-refs (if db-before - (set (map :db/id (:block/path-refs (d/entity db-before (:db/id block))))) - #{}) - new-refs (set (util/concat-without-nil - [(:db/id (:block/page block))] - (map :db/id (:block/refs block)) - parents-refs)) - refs-changed? (not= old-refs new-refs) - children (db-model/get-block-children-ids repo (:block/uuid block)) - ;; Builds map of children ids to their parent id and :block/refs ids - children-maps (into {} - (map (fn [id] - (let [entity (db/entity [:block/uuid id])] - [(:db/id entity) - {:parent-id (get-in entity [:block/parent :db/id]) - :block-ref-ids (map :db/id (:block/refs entity))}])) - children)) - children-refs (map (fn [[id {:keys [block-ref-ids] :as child-map}]] - {:db/id id - ;; Recalculate :block/path-refs as db contains stale data for this attribute - :block/path-refs - (set/union - ;; Refs from top-level parent - new-refs - ;; Refs from current block - block-ref-ids - ;; Refs from parents in between top-level - ;; parent and current block - (loop [parent-refs #{} - parent-id (:parent-id child-map)] - (if-let [parent (children-maps parent-id)] - (recur (into parent-refs (:block-ref-ids parent)) - (:parent-id parent)) - ;; exits when top-level parent is reached - parent-refs)))}) - children-maps)] - (swap! *computed-ids set/union (set (cons (:block/uuid block) children))) - (util/concat-without-nil - [(when (and (seq new-refs) - refs-changed?) - {:db/id (:db/id block) - :block/path-refs new-refs})] - children-refs)))) - blocks)))))) + [{:keys [tx-meta] :as tx-report} blocks] + (when (and (:outliner-op tx-meta) (react/path-refs-need-recalculated? tx-meta)) + (outliner-pipeline/computer-block-path-refs tx-report blocks))) (defn invoke-hooks [tx-report] diff --git a/src/test/fixtures/broken-outliner-data-with-cycle.edn b/src/test/fixtures/broken-outliner-data-with-cycle.edn deleted file mode 100644 index 32a49b4da..000000000 --- a/src/test/fixtures/broken-outliner-data-with-cycle.edn +++ /dev/null @@ -1,14 +0,0 @@ -;;; datoms -;;; - 1 <----+ -;;; - 2 | -;;; - 3 -+ -[{:db/id 1 - :block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4" - :block/parent 3} - {:db/id 2 - :block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5" - :block/parent 1} - {:db/id 3 - :block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2" - :block/parent 2} - ] diff --git a/src/test/frontend/db/model_test.cljs b/src/test/frontend/db/model_test.cljs index 39b577856..83915572d 100644 --- a/src/test/frontend/db/model_test.cljs +++ b/src/test/frontend/db/model_test.cljs @@ -3,12 +3,9 @@ [frontend.db.model :as model] [frontend.db :as db] [frontend.db.conn :as conn] - [logseq.db.schema :as db-schema] [frontend.test.helper :as test-helper :refer [load-test-files]] [datascript.core :as d] - [shadow.resource :as rc] - [clojure.set :as set] - [clojure.edn :as edn])) + [clojure.set :as set])) (use-fixtures :each {:before test-helper/start-test-db! :after test-helper/destroy-test-db!}) @@ -151,18 +148,6 @@ foo:: bar"}]) "Non header block's content returns nil")) -(def broken-outliner-data-with-cycle (-> (rc/inline "fixtures/broken-outliner-data-with-cycle.edn") - edn/read-string)) - -(deftest get-block-children-ids-on-bad-outliner-data - (let [db (d/db-with (d/empty-db db-schema/schema) - broken-outliner-data-with-cycle)] - - (is (= "bad outliner data, need to re-index to fix" - (try (model/get-block-children-ids-in-db db #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4") - (catch :default e - (ex-message e))))))) - (deftest get-block-immediate-children (load-test-files [{:file/path "pages/page1.md" :file/content "\n diff --git a/src/test/frontend/modules/outliner/pipeline_test.cljs b/src/test/frontend/modules/outliner/pipeline_test.cljs index 2dcca1b49..d5038b9be 100644 --- a/src/test/frontend/modules/outliner/pipeline_test.cljs +++ b/src/test/frontend/modules/outliner/pipeline_test.cljs @@ -14,6 +14,7 @@ db) (map first))) +;; TODO: Move this test to outliner dep when there is a load-test-files helper for deps (deftest compute-block-path-refs (load-test-files [{:file/path "pages/page1.md" :file/content "prop:: #bar @@ -31,7 +32,7 @@ :block/path-refs [{:db/id new-tag-id}]) %) blocks) - refs-tx (pipeline/compute-block-path-refs {:tx-meta {:outliner-op :save-block}} modified-blocks) + refs-tx (pipeline/compute-block-path-refs {:tx-meta {:outliner-op :save-block} :db-after @conn} modified-blocks) _ (d/transact! conn (concat (map (fn [m] [:db/retract (:db/id m) :block/path-refs]) refs-tx) refs-tx)) updated-blocks (->> (get-blocks @conn)