From 6ad34f5b3fc10e66633d420c4ccd0ce528a029b8 Mon Sep 17 00:00:00 2001 From: defclass Date: Thu, 17 Dec 2020 14:23:52 +0800 Subject: [PATCH] refactor(db): move some fn into react --- src/main/frontend/db.cljs | 10 +- src/main/frontend/db/base.cljs | 59 +++++ src/main/frontend/db/debug.cljs | 5 +- src/main/frontend/db/model.cljs | 341 ++++++------------------- src/main/frontend/db/query_custom.cljs | 5 +- src/main/frontend/db/react.cljs | 169 ++++++++++-- src/main/frontend/db_mixins.cljs | 2 +- src/main/frontend/handler/repo.cljs | 50 ++++ 8 files changed, 351 insertions(+), 290 deletions(-) create mode 100644 src/main/frontend/db/base.cljs diff --git a/src/main/frontend/db.cljs b/src/main/frontend/db.cljs index 997e623c5..311500979 100644 --- a/src/main/frontend/db.cljs +++ b/src/main/frontend/db.cljs @@ -32,14 +32,16 @@ string->db with-repo] [frontend.db.model - entity pull pull-many - add-properties! add-q! add-query-component! block-and-children-transform blocks-count blocks-count-cache clean-export! clear-query-state! clear-query-state-without-refs-and-embeds! cloned? delete-blocks delete-file! delete-file-blocks! delete-file-pages! delete-file-tx delete-files delete-pages-by-files filter-only-public-pages-and-blocks get-alias-page get-all-block-contents get-all-tagged-pages get-all-tags get-all-templates get-block-and-children get-block-and-children-no-cache get-block-blocks-cache-atom get-block-by-uuid get-block-children get-block-children-ids get-block-content get-block-file get-block-immediate-children get-block-page get-block-page-end-pos get-block-parent get-block-parents get-block-referenced-blocks get-block-refs-count get-blocks-by-priority get-blocks-contents get-collapsed-blocks get-config get-custom-css get-date-scheduled-or-deadlines get-db-type get-empty-pages get-file get-file-after-blocks get-file-after-blocks-meta get-file-blocks get-file-contents get-file-last-modified-at get-file-no-sub get-file-page get-file-page-id get-file-pages get-files get-files-blocks get-files-full get-files-that-referenced-page get-journals-length get-key-value get-latest-journals get-marker-blocks get-matched-blocks get-page get-page-alias get-page-alias-names get-page-blocks get-page-blocks-cache-atom get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-name get-page-properties get-page-properties-content get-page-referenced-blocks get-page-referenced-pages get-page-unlinked-references get-pages get-pages-relation get-pages-that-mentioned-page get-pages-with-modified-at get-public-pages get-tag-pages journal-page? kv local-native-fs? mark-repo-as-cloned! page-alias-set page-blocks-transform pull-block q query-components query-entity-in-component query-state rebuild-page-blocks-children remove-custom-query! remove-key! remove-q! remove-query-component! reset-config! set-file-last-modified-at! set-new-result! sub-key-value template-exists? transact! transact-files-db! with-block-refs-count] + add-properties! block-and-children-transform blocks-count blocks-count-cache clean-export! cloned? delete-blocks delete-file! delete-file-blocks! delete-file-pages! delete-file-tx delete-files delete-pages-by-files filter-only-public-pages-and-blocks get-alias-page get-all-block-contents get-all-tagged-pages get-all-tags get-all-templates get-block-and-children get-block-and-children-no-cache get-block-by-uuid get-block-children get-block-children-ids get-block-content get-block-file get-block-immediate-children get-block-page get-block-page-end-pos get-block-parent get-block-parents get-block-referenced-blocks get-block-refs-count get-blocks-by-priority get-blocks-contents get-collapsed-blocks get-config get-custom-css get-date-scheduled-or-deadlines get-db-type get-empty-pages get-file get-file-after-blocks get-file-after-blocks-meta get-file-blocks get-file-contents get-file-last-modified-at get-file-no-sub get-file-page get-file-page-id get-file-pages get-files get-files-blocks get-files-full get-files-that-referenced-page get-journals-length get-latest-journals get-marker-blocks get-matched-blocks get-page get-page-alias get-page-alias-names get-page-blocks get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-name get-page-properties get-page-properties-content get-page-referenced-blocks get-page-referenced-pages get-page-unlinked-references get-pages get-pages-relation get-pages-that-mentioned-page get-pages-with-modified-at get-public-pages get-tag-pages journal-page? local-native-fs? mark-repo-as-cloned! page-alias-set page-blocks-transform pull-block rebuild-page-blocks-children reset-config! set-file-last-modified-at! sub-key-value template-exists? transact-files-db! with-block-refs-count] [frontend.db.react - get-current-marker get-current-page get-current-priority get-handler-keys set-file-content! set-key-value transact-react!] + get-current-marker get-current-page get-current-priority get-handler-keys set-file-content! set-key-value transact-react! remove-key! remove-q! remove-query-component! add-q! add-query-component! clear-query-state! clear-query-state-without-refs-and-embeds! get-block-blocks-cache-atom get-page-blocks-cache-atom kv q query-state query-components query-entity-in-component remove-custom-query! set-new-result!] [frontend.db.query-custom - custom-query custom-query-result-transform]) + custom-query custom-query-result-transform] + + [frontend.db.base + entity pull pull-many transact! get-key-value]) ;; persisting DBs between page reloads (defn persist! [repo] diff --git a/src/main/frontend/db/base.cljs b/src/main/frontend/db/base.cljs new file mode 100644 index 000000000..77ca63c03 --- /dev/null +++ b/src/main/frontend/db/base.cljs @@ -0,0 +1,59 @@ +(ns frontend.db.base + "Base query utils are required by model.cljs and react.cljs" + (:require [frontend.state :as state] + [frontend.db.conn :as conn] + [datascript.core :as d] + [frontend.config :as config] + [frontend.util :as util])) + +(defn entity + ([id-or-lookup-ref] + (entity (state/get-current-repo) id-or-lookup-ref)) + ([repo id-or-lookup-ref] + (when-let [db (conn/get-conn repo)] + (d/entity db id-or-lookup-ref)))) + +(defn pull + ([eid] + (pull (state/get-current-repo) '[*] eid)) + ([selector eid] + (pull (state/get-current-repo) selector eid)) + ([repo selector eid] + (when-let [conn (conn/get-conn repo)] + (try + (d/pull conn + selector + eid) + (catch js/Error e + nil))))) + +(defn pull-many + ([eids] + (pull-many '[*] eids)) + ([selector eids] + (pull-many (state/get-current-repo) selector eids)) + ([repo selector eids] + (when-let [conn (conn/get-conn repo)] + (try + (d/pull-many conn selector eids) + (catch js/Error e + (js/console.error e)))))) + +(defn transact! + ([tx-data] + (transact! (state/get-current-repo) tx-data)) + ([repo-url tx-data] + (when-not config/publishing? + (let [tx-data (->> (util/remove-nils tx-data) + (remove nil?))] + (when (seq tx-data) + (when-let [conn (conn/get-conn repo-url false)] + (d/transact! conn (vec tx-data)))))))) + +(defn get-key-value + ([key] + (get-key-value (state/get-current-repo) key)) + ([repo-url key] + (when-let [db (conn/get-conn repo-url)] + (some-> (d/entity db key) + key)))) diff --git a/src/main/frontend/db/debug.cljs b/src/main/frontend/db/debug.cljs index 70106c934..be4be2c70 100644 --- a/src/main/frontend/db/debug.cljs +++ b/src/main/frontend/db/debug.cljs @@ -1,11 +1,12 @@ (ns frontend.db.debug (:require [medley.core :as medley] - [frontend.db.model :as model])) + [frontend.db.model :as model] + [frontend.db.base :as base])) ;; shortcut for query a block with string ref (defn qb [string-id] - (model/pull [:block/uuid (medley/uuid string-id)])) + (base/pull [:block/uuid (medley/uuid string-id)])) (comment (defn debug! diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index c9dea9a67..de6cbfe93 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -2,6 +2,7 @@ "Core db functions." (:require [frontend.db.conn :as conn] [frontend.db.utils :as db-utils] + [frontend.db.react :as react] [datascript.core :as d] [frontend.date :as date] [medley.core :as medley] @@ -16,224 +17,30 @@ [cljs-time.core :as t] [cljs-time.coerce :as tc] [frontend.util :as util :refer [react] :refer-macros [profile]] - [frontend.db-schema :as db-schema])) + [frontend.db-schema :as db-schema] + [frontend.db.base :as base])) ;; TODO: extract to specific models and move data transform logic to the ;; correponding handlers. -(defn entity - ([id-or-lookup-ref] - (entity (state/get-current-repo) id-or-lookup-ref)) - ([repo id-or-lookup-ref] - (when-let [db (conn/get-conn repo)] - (d/entity db id-or-lookup-ref)))) - -(defn pull - ([eid] - (pull (state/get-current-repo) '[*] eid)) - ([selector eid] - (pull (state/get-current-repo) selector eid)) - ([repo selector eid] - (when-let [conn (conn/get-conn repo)] - (try - (d/pull conn - selector - eid) - (catch js/Error e - nil))))) - -(defn pull-many - ([eids] - (pull-many '[*] eids)) - ([selector eids] - (pull-many (state/get-current-repo) selector eids)) - ([repo selector eids] - (when-let [conn (conn/get-conn repo)] - (try - (d/pull-many conn selector eids) - (catch js/Error e - (js/console.error e)))))) - -(defn transact! - ([tx-data] - (transact! (state/get-current-repo) tx-data)) - ([repo-url tx-data] - (when-not config/publishing? - (let [tx-data (->> (util/remove-nils tx-data) - (remove nil?))] - (when (seq tx-data) - (when-let [conn (conn/get-conn repo-url false)] - (d/transact! conn (vec tx-data)))))))) - (defn transact-files-db! ([tx-data] - (transact! (state/get-current-repo) tx-data)) + (base/transact! (state/get-current-repo) tx-data)) ([repo-url tx-data] (when-not config/publishing? (let [tx-data (->> (util/remove-nils tx-data) - (remove nil?) - (map #(dissoc % :file/handle :file/type)))] + (remove nil?) + (map #(dissoc % :file/handle :file/type)))] (when (seq tx-data) (when-let [conn (conn/get-files-conn repo-url)] (d/transact! conn (vec tx-data)))))))) -;; Query atom of map of Key ([repo q inputs]) -> atom -;; TODO: replace with LRUCache, only keep the latest 20 or 50 items? -(defonce query-state (atom {})) - -(def ^:dynamic *query-component*) - -;; key -> components -(defonce query-components (atom {})) - -(defonce blocks-count-cache (atom nil)) - -(defn set-new-result! - [k new-result] - (when-let [result-atom (get-in @query-state [k :result])] - (reset! result-atom new-result))) - -;; KV -(defn get-key-value - ([key] - (get-key-value (state/get-current-repo) key)) - ([repo-url key] - (when-let [db (conn/get-conn repo-url)] - (some-> (d/entity db key) - key)))) - -(defn kv - [key value] - {:db/id -1 - :db/ident key - key value}) - -(defn remove-key! - [repo-url key] - (transact! repo-url [[:db.fn/retractEntity [:db/ident key]]]) - (set-new-result! [repo-url :kv key] nil)) - -(defn clear-query-state! - [] - (reset! query-state {})) - -;; remove block refs, block embeds, page embeds -(defn clear-query-state-without-refs-and-embeds! - [] - (let [state @query-state - state (->> (filter (fn [[[_repo k] v]] - (contains? #{:blocks :block/block :custom} k)) state) - (into {}))] - (reset! query-state state))) - -;; TODO: Add components which subscribed to a specific query -(defn add-q! - [k query inputs result-atom transform-fn query-fn inputs-fn] - (swap! query-state assoc k {:query query - :inputs inputs - :result result-atom - :transform-fn transform-fn - :query-fn query-fn - :inputs-fn inputs-fn}) - result-atom) - -(defn remove-q! - [k] - (swap! query-state dissoc k)) - -(defn add-query-component! - [key component] - (swap! query-components update key - (fn [components] - (distinct (conj components component))))) - -(defn remove-query-component! - [component] - (reset! - query-components - (->> (for [[k components] @query-components - :let [new-components (remove #(= component %) components)]] - (if (empty? new-components) ; no subscribed components - (do (remove-q! k) - nil) - [k new-components])) - (keep identity) - (into {})))) - -(defn get-page-blocks-cache-atom - [repo page-id] - (:result (get @query-state [repo :page/blocks page-id]))) - -(defn get-block-blocks-cache-atom - [repo block-id] - (:result (get @query-state [repo :block/block block-id]))) - -;; TODO: rename :custom to :query/custom -(defn remove-custom-query! - [repo query] - (remove-q! [repo :custom query])) - -;; Reactive query - - -(defn query-entity-in-component - ([id-or-lookup-ref] - (entity (state/get-current-repo) id-or-lookup-ref)) - ([repo id-or-lookup-ref] - (let [k [:entity id-or-lookup-ref] - result-atom (:result (get @query-state k))] - (when-let [component *query-component*] - (add-query-component! k component)) - (when-let [db (conn/get-conn repo)] - (let [result (d/entity db id-or-lookup-ref) - result-atom (or result-atom (atom nil))] - (set! (.-state result-atom) result) - (add-q! k nil nil result-atom identity identity identity)))))) - -(defn q - [repo k {:keys [use-cache? files-db? transform-fn query-fn inputs-fn] - :or {use-cache? true - files-db? false - transform-fn identity}} query & inputs] - (let [kv? (and (vector? k) (= :kv (first k))) - k (vec (cons repo k))] - (when-let [conn (if files-db? - (when-let [files-conn (conn/get-files-conn repo)] - (deref files-conn)) - (conn/get-conn repo))] - (let [result-atom (:result (get @query-state k))] - (when-let [component *query-component*] - (add-query-component! k component)) - (if (and use-cache? result-atom) - result-atom - (let [result (cond - query-fn - (query-fn conn) - - inputs-fn - (let [inputs (inputs-fn)] - (apply d/q query conn inputs)) - - kv? - (d/entity conn (last k)) - - (seq inputs) - (apply d/q query conn inputs) - - :else - (d/q query conn)) - result (transform-fn result) - result-atom (or result-atom (atom nil))] - ;; Don't notify watches now - (set! (.-state result-atom) result) - (add-q! k query inputs result-atom transform-fn query-fn inputs-fn))))))) - (defn sub-key-value ([key] (sub-key-value (state/get-current-repo) key)) ([repo-url key] (when (conn/get-conn repo-url) - (-> (q repo-url [:kv key] {} key key) + (-> (react/q repo-url [:kv key] {} key key) react key)))) @@ -242,8 +49,8 @@ (let [repo (state/get-current-repo)] (when (conn/get-conn repo) (-> - (q repo [:blocks id] {} - '[:find (pull ?block [*]) + (react/q repo [:blocks id] {} + '[:find (base/pull ?block [*]) :in $ ?id :where [?block :block/uuid ?id]] @@ -256,7 +63,7 @@ (let [repo (state/get-current-repo)] (when (conn/get-conn repo) (some->> - (q repo [:tags] {} + (react/q repo [:tags] {} '[:find ?name ?h ?p :where [?t :tag/name ?name] @@ -344,7 +151,7 @@ [repo page-name] (let [alias-ids (get-page-alias repo page-name)] (when (seq alias-ids) - (->> (pull-many repo '[:page/name] alias-ids) + (->> (base/pull-many repo '[:page/name] alias-ids) (map :page/name) distinct)))) @@ -390,7 +197,7 @@ (when end-pos (let [pred (fn [db meta] (>= (:start-pos meta) end-pos))] - (-> (d/q '[:find (pull ?block [*]) + (-> (d/q '[:find (base/pull ?block [*]) :in $ ?file-id ?pred :where [?block :block/file ?file-id] @@ -410,7 +217,7 @@ ks (if content-level? '[:block/uuid :block/meta :block/content :block/level] '[:block/uuid :block/meta]) - blocks (pull-many repo-url ks eids)] + blocks (base/pull-many repo-url ks eids)] (->> (filter (fn [{:block/keys [meta]}] (>= (:start-pos meta) end-pos)) blocks) db-utils/sort-by-pos)))) @@ -446,7 +253,7 @@ ([repo path] (when (and repo path) (-> - (q repo [:file/content path] + (react/q repo [:file/content path] {:files-db? true :use-cache? true} '[:find ?content @@ -475,7 +282,7 @@ (when-let [conn (conn/get-files-conn repo)] (->> (d/q - '[:find (pull ?file [*]) + '[:find (base/pull ?file [*]) :where [?file :file/path]] @conn) @@ -504,17 +311,17 @@ (defn get-block-by-uuid [uuid] - (entity [:block/uuid uuid])) + (base/entity [:block/uuid uuid])) (defn get-page-format [page-name] - (when-let [file (:page/file (entity [:page/name page-name]))] - (when-let [path (:file/path (entity (:db/id file)))] + (when-let [file (:page/file (base/entity [:page/name page-name]))] + (when-let [path (:file/path (base/entity (:db/id file)))] (format/get-format path)))) (defn page-alias-set [repo-url page] - (when-let [page-id (:db/id (entity [:page/name page]))] + (when-let [page-id (:db/id (base/entity [:page/name page]))] (let [aliases (get-page-alias repo-url page) aliases (if (seq aliases) (set @@ -553,7 +360,7 @@ (defn sort-blocks [blocks] (let [pages-ids (map (comp :db/id :block/page) blocks) - pages (pull-many '[:db/id :page/last-modified-at :page/name :page/original-name] pages-ids) + pages (base/pull-many '[:db/id :page/last-modified-at :page/name :page/original-name] pages-ids) pages-map (reduce (fn [acc p] (assoc acc (:db/id p) p)) {} pages) blocks (map (fn [block] @@ -566,9 +373,9 @@ [repo-url marker] (let [marker (string/upper-case marker)] (some->> - (q repo-url [:marker/blocks marker] + (react/q repo-url [:marker/blocks marker] {:use-cache? true} - '[:find (pull ?h [*]) + '[:find (base/pull ?h [*]) :in $ ?marker :where [?h :block/marker ?m] @@ -584,7 +391,7 @@ (defn get-page-properties [page] - (when-let [page (entity [:page/name page])] + (when-let [page (base/entity [:page/name page])] (:page/properties page))) (defn add-properties! @@ -636,18 +443,18 @@ :or {use-cache? true pull-keys '[*]}}] (let [page (string/lower-case page) - page-id (or (:db/id (entity repo-url [:page/name page])) - (:db/id (entity repo-url [:page/original-name page]))) + page-id (or (:db/id (base/entity repo-url [:page/name page])) + (:db/id (base/entity repo-url [:page/original-name page]))) db (conn/get-conn repo-url)] (when page-id (some-> - (q repo-url [:page/blocks page-id] + (react/q repo-url [:page/blocks page-id] {:use-cache? use-cache? :transform-fn #(page-blocks-transform repo-url %) :query-fn (fn [db] (let [datoms (d/datoms db :avet :block/page page-id) block-eids (mapv :e datoms)] - (pull-many repo-url pull-keys block-eids)))} + (base/pull-many repo-url pull-keys block-eids)))} nil) react))))) @@ -659,13 +466,13 @@ ([repo-url page {:keys [pull-keys] :or {pull-keys '[*]}}] (let [page (string/lower-case page) - page-id (or (:db/id (entity repo-url [:page/name page])) - (:db/id (entity repo-url [:page/original-name page]))) + page-id (or (:db/id (base/entity repo-url [:page/name page])) + (:db/id (base/entity repo-url [:page/original-name page]))) db (conn/get-conn repo-url)] (when page-id (let [datoms (d/datoms db :avet :block/page page-id) block-eids (mapv :e datoms)] - (some->> (pull-many repo-url pull-keys block-eids) + (some->> (base/pull-many repo-url pull-keys block-eids) (page-blocks-transform repo-url))))))) (defn get-page-blocks-count @@ -693,18 +500,18 @@ (defn get-block-page [repo block-id] - (when-let [block (entity repo [:block/uuid block-id])] - (entity repo (:db/id (:block/page block))))) + (when-let [block (base/entity repo [:block/uuid block-id])] + (base/entity repo (:db/id (:block/page block))))) (defn get-block-page-end-pos [repo page-name] (or - (when-let [page-id (:db/id (entity repo [:page/name (string/lower-case page-name)]))] + (when-let [page-id (:db/id (base/entity repo [:page/name (string/lower-case page-name)]))] (when-let [db (conn/get-conn repo)] (let [block-eids (->> (d/datoms db :avet :block/page page-id) (mapv :e))] (when (seq block-eids) - (let [blocks (pull-many repo '[:block/meta] block-eids)] + (let [blocks (base/pull-many repo '[:block/meta] block-eids)] (-> (last (db-utils/sort-by-pos blocks)) (get-in [:block/meta :end-pos]))))))) ;; TODO: need more thoughts @@ -714,8 +521,8 @@ [repo priority] (let [priority (string/capitalize priority)] (when (conn/get-conn repo) - (->> (q repo [:priority/blocks priority] {} - '[:find (pull ?h [*]) + (->> (react/q repo [:priority/blocks priority] {} + '[:find (base/pull ?h [*]) :in $ ?priority :where [?h :block/priority ?priority]] @@ -761,7 +568,7 @@ (defn get-block-children-ids [repo block-uuid] (when-let [conn (conn/get-conn repo)] - (let [eid (:db/id (entity repo [:block/uuid block-uuid]))] + (let [eid (:db/id (base/entity repo [:block/uuid block-uuid]))] (->> (d/q '[:find ?e1 :in $ ?e2 % @@ -779,37 +586,37 @@ (defn get-block-immediate-children [repo block-uuid] (when-let [conn (conn/get-conn repo)] - (let [ids (->> (:block/children (entity repo [:block/uuid block-uuid])) + (let [ids (->> (:block/children (base/entity repo [:block/uuid block-uuid])) (map :db/id))] (when (seq ids) - (pull-many repo '[*] ids))))) + (base/pull-many repo '[*] ids))))) (defn get-block-children [repo block-uuid] (when-let [conn (conn/get-conn repo)] (let [ids (get-block-children-ids repo block-uuid)] (when (seq ids) - (pull-many repo '[*] ids))))) + (base/pull-many repo '[*] ids))))) (defn get-block-and-children ([repo block-uuid] (get-block-and-children repo block-uuid true)) ([repo block-uuid use-cache?] - (let [block (entity repo [:block/uuid block-uuid]) + (let [block (base/entity repo [:block/uuid block-uuid]) page (:db/id (:block/page block)) pos (:start-pos (:block/meta block)) level (:block/level block) pred (fn [] - (let [block (entity repo [:block/uuid block-uuid]) + (let [block (base/entity repo [:block/uuid block-uuid]) pos (:start-pos (:block/meta block))] (fn [data meta] (>= (:start-pos meta) pos))))] - (some-> (q repo [:block/block block-uuid] + (some-> (react/q repo [:block/block block-uuid] {:use-cache? use-cache? :transform-fn #(block-and-children-transform % repo block-uuid level) :inputs-fn (fn [] [page (pred)])} - '[:find (pull ?block [*]) + '[:find (base/pull ?block [*]) :in $ ?page ?pred :where [?block :block/page ?page] @@ -820,14 +627,14 @@ ;; TODO: performance (defn get-block-and-children-no-cache [repo block-uuid] - (let [block (entity repo [:block/uuid block-uuid]) + (let [block (base/entity repo [:block/uuid block-uuid]) page (:db/id (:block/page block)) pos (:start-pos (:block/meta block)) level (:block/level block) pred (fn [data meta] (>= (:start-pos meta) pos))] (-> (d/q - '[:find (pull ?block [*]) + '[:find (base/pull ?block [*]) :in $ ?page ?pred :where [?block :block/page ?page] @@ -865,15 +672,15 @@ (defn get-page-file [page-name] - (some-> (entity [:page/name page-name]) + (some-> (base/entity [:page/name page-name]) :page/file)) (defn get-block-file [block-id] - (let [page-id (some-> (entity [:block/uuid block-id]) + (let [page-id (some-> (base/entity [:block/uuid block-id]) :block/page :db/id)] - (:page/file (entity page-id)))) + (:page/file (base/entity page-id)))) (defn get-file-page-id [file-path] @@ -893,8 +700,8 @@ (defn get-page [page-name] (if (util/uuid-string? page-name) - (entity [:block/uuid (uuid page-name)]) - (entity [:page/name page-name]))) + (base/entity [:block/uuid (uuid page-name)]) + (base/entity [:page/name page-name]))) (defn get-page-name [file ast] @@ -951,7 +758,7 @@ _ (.setDate date (- (.getDate date) (dec n))) today (db-utils/date->int (js/Date.)) pages (->> - (q repo-url [:journals] {:use-cache? false} + (react/q repo-url [:journals] {:use-cache? false} '[:find ?page-name ?journal-day :in $ ?today :where @@ -976,8 +783,8 @@ [repo page] (when (conn/get-conn repo) (let [pages (page-alias-set repo page) - page-id (:db/id (entity [:page/name page])) - ref-pages (->> (q repo [:page/ref-pages page-id] {:use-cache? false} + page-id (:db/id (base/entity [:page/name page])) + ref-pages (->> (react/q repo [:page/ref-pages page-id] {:use-cache? false} '[:find ?ref-page-name :in $ ?pages :where @@ -1030,9 +837,9 @@ (defn get-pages-that-mentioned-page [repo page] (when (conn/get-conn repo) - (let [page-id (:db/id (entity [:page/name page])) + (let [page-id (:db/id (base/entity [:page/name page])) pages (page-alias-set repo page) - mentioned-pages (->> (q repo [:page/mentioned-pages page-id] {:use-cache? false} + mentioned-pages (->> (react/q repo [:page/mentioned-pages page-id] {:use-cache? false} '[:find ?mentioned-page-name :in $ ?pages ?page-name :where @@ -1050,10 +857,10 @@ [page] (when-let [repo (state/get-current-repo)] (when (conn/get-conn repo) - (let [page-id (:db/id (entity [:page/name page])) + (let [page-id (:db/id (base/entity [:page/name page])) pages (page-alias-set repo page)] - (->> (q repo [:page/refed-blocks page-id] {} - '[:find (pull ?block [*]) + (->> (react/q repo [:page/refed-blocks page-id] {} + '[:find (base/pull ?block [*]) :in $ ?pages :where [?block :block/ref-pages ?ref-page] @@ -1072,8 +879,8 @@ (when-let [date (date/journal-title->int journal-title)] (when-let [repo (state/get-current-repo)] (when-let [conn (conn/get-conn repo)] - (->> (q repo [:custom :scheduled-deadline journal-title] {} - '[:find (pull ?block [*]) + (->> (react/q repo [:custom :scheduled-deadline journal-title] {} + '[:find (base/pull ?block [*]) :in $ ?day :where (or @@ -1107,11 +914,11 @@ [page] (when-let [repo (state/get-current-repo)] (when-let [conn (conn/get-conn repo)] - (let [page-id (:db/id (entity [:page/name page])) + (let [page-id (:db/id (base/entity [:page/name page])) pages (page-alias-set repo page) pattern (re-pattern (str "(?i)" page))] (->> (d/q - '[:find (pull ?block [*]) + '[:find (base/pull ?block [*]) :in $ ?pattern :where [?block :block/content ?content] @@ -1133,8 +940,8 @@ [block-uuid] (when-let [repo (state/get-current-repo)] (when (conn/get-conn repo) - (->> (q repo [:block/refed-blocks block-uuid] {} - '[:find (pull ?ref-block [*]) + (->> (react/q repo [:block/refed-blocks block-uuid] {} + '[:find (base/pull ?ref-block [*]) :in $ ?block-uuid :where [?block :block/uuid ?block-uuid] @@ -1160,7 +967,7 @@ pred) (take limit) db-utils/seq-flatten - (pull-many '[:block/uuid + (base/pull-many '[:block/uuid :block/content :block/properties :block/format @@ -1170,16 +977,16 @@ (defn get-blocks-contents [repo block-uuids] (let [db (conn/get-conn repo)] - (pull-many repo '[:block/content] + (base/pull-many repo '[:block/content] (mapv (fn [id] [:block/uuid id]) block-uuids)))) (defn journal-page? [page-name] - (:page/journal? (entity [:page/name page-name]))) + (:page/journal? (base/entity [:page/name page-name]))) (defn mark-repo-as-cloned! [repo-url] - (transact! + (base/transact! [{:repo/url repo-url :repo/cloned? true}])) @@ -1214,7 +1021,7 @@ (defn get-db-type [repo] - (get-key-value repo :db/type)) + (base/get-key-value repo :db/type)) (defn local-native-fs? [repo] @@ -1314,6 +1121,8 @@ (let [templates (map string/lower-case templates)] (contains? (set templates) (string/lower-case title))))))) +(defonce blocks-count-cache (atom nil)) + (defn blocks-count ([] (blocks-count true)) @@ -1331,7 +1140,7 @@ (->> (d/datoms conn :avet :block/uuid) (map :v) (map (fn [id] - (let [e (entity [:block/uuid id])] + (let [e (base/entity [:block/uuid id])] {:db/id (:db/id e) :block/uuid id :block/content (:block/content e) @@ -1350,7 +1159,7 @@ (defn filter-only-public-pages-and-blocks [db] (let [public-pages (get-public-pages db) - contents-id (:db/id (entity [:page/name "contents"]))] + contents-id (:db/id (base/entity [:page/name "contents"]))] (when (seq public-pages) (let [public-pages (set (conj public-pages contents-id)) page-or-block? #(contains? #{"page" "block" "me" "recent" "file"} %) @@ -1397,7 +1206,7 @@ (defn delete-file! [repo-url file-path] - (transact! repo-url (delete-file-tx repo-url file-path))) + (base/transact! repo-url (delete-file-tx repo-url file-path))) (defn delete-pages-by-files [files] diff --git a/src/main/frontend/db/query_custom.cljs b/src/main/frontend/db/query_custom.cljs index cd203abb5..1410b8592 100644 --- a/src/main/frontend/db/query_custom.cljs +++ b/src/main/frontend/db/query_custom.cljs @@ -9,7 +9,8 @@ [cljs.reader :as reader] [frontend.extensions.sci :as sci] [lambdaisland.glogi :as log] - [frontend.util :as util])) + [frontend.util :as util] + [frontend.db.react :as react])) (defn- resolve-input [input] @@ -42,7 +43,7 @@ (let [inputs (map resolve-input inputs) repo (state/get-current-repo) k [:custom query']] - (apply model/q repo k query-opts query inputs)) + (apply react/q repo k query-opts query inputs)) (catch js/Error e (println "Custom query failed: ") (js/console.dir e)))) diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index abc82af5f..d24472a5f 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -5,20 +5,159 @@ solution. " (:require [frontend.db.conn :as conn] - [frontend.db.utils :as db-utils] - [frontend.db.model :as model] + [frontend.db.base :as base] [frontend.state :as state] [frontend.date :as date] [frontend.util :as util :refer-macros [profile] :refer [react]] [clojure.string :as string] [frontend.config :as config] - [frontend.format :as format] - [cljs-time.core :as t] - [cljs-time.coerce :as tc] - [frontend.utf8 :as utf8] [datascript.core :as d] [lambdaisland.glogi :as log])) +;; Query atom of map of Key ([repo q inputs]) -> atom +;; TODO: replace with LRUCache, only keep the latest 20 or 50 items? + +(defonce query-state (atom {})) + +(def ^:dynamic *query-component*) + +;; key -> components +(defonce query-components (atom {})) + +(defn set-new-result! + [k new-result] + (when-let [result-atom (get-in @query-state [k :result])] + (reset! result-atom new-result))) + +;; KV + +(defn kv + [key value] + {:db/id -1 + :db/ident key + key value}) + +(defn remove-key! + [repo-url key] + (base/transact! repo-url [[:db.fn/retractEntity [:db/ident key]]]) + (set-new-result! [repo-url :kv key] nil)) + +(defn clear-query-state! + [] + (reset! query-state {})) + +(defn clear-query-state-without-refs-and-embeds! + [] + (let [state @query-state + state (->> (filter (fn [[[_repo k] v]] + (contains? #{:blocks :block/block :custom} k)) state) + (into {}))] + (reset! query-state state))) + +;; TODO: Add components which subscribed to a specific query +(defn add-q! + [k query inputs result-atom transform-fn query-fn inputs-fn] + (swap! query-state assoc k {:query query + :inputs inputs + :result result-atom + :transform-fn transform-fn + :query-fn query-fn + :inputs-fn inputs-fn}) + result-atom) + +(defn remove-q! + [k] + (swap! query-state dissoc k)) + +(defn add-query-component! + [key component] + (swap! query-components update key + (fn [components] + (distinct (conj components component))))) + +(defn remove-query-component! + [component] + (reset! + query-components + (->> (for [[k components] @query-components + :let [new-components (remove #(= component %) components)]] + (if (empty? new-components) ; no subscribed components + (do (remove-q! k) + nil) + [k new-components])) + (keep identity) + (into {})))) + +(defn get-page-blocks-cache-atom + [repo page-id] + (:result (get @query-state [repo :page/blocks page-id]))) + +(defn get-block-blocks-cache-atom + [repo block-id] + (:result (get @query-state [repo :block/block block-id]))) + +;; TODO: rename :custom to :query/custom +(defn remove-custom-query! + [repo query] + (remove-q! [repo :custom query])) + +;; Reactive query + + +(defn query-entity-in-component + ([id-or-lookup-ref] + (base/entity (state/get-current-repo) id-or-lookup-ref)) + ([repo id-or-lookup-ref] + (let [k [:entity id-or-lookup-ref] + result-atom (:result (get @query-state k))] + (when-let [component *query-component*] + (add-query-component! k component)) + (when-let [db (conn/get-conn repo)] + (let [result (d/entity db id-or-lookup-ref) + result-atom (or result-atom (atom nil))] + (set! (.-state result-atom) result) + (add-q! k nil nil result-atom identity identity identity)))))) + +(defn q + [repo k {:keys [use-cache? files-db? transform-fn query-fn inputs-fn] + :or {use-cache? true + files-db? false + transform-fn identity}} query & inputs] + (let [kv? (and (vector? k) (= :kv (first k))) + k (vec (cons repo k))] + (when-let [conn (if files-db? + (when-let [files-conn (conn/get-files-conn repo)] + (deref files-conn)) + (conn/get-conn repo))] + (let [result-atom (:result (get @query-state k))] + (when-let [component *query-component*] + (add-query-component! k component)) + (if (and use-cache? result-atom) + result-atom + (let [result (cond + query-fn + (query-fn conn) + + inputs-fn + (let [inputs (inputs-fn)] + (apply d/q query conn inputs)) + + kv? + (d/entity conn (last k)) + + (seq inputs) + (apply d/q query conn inputs) + + :else + (d/q query conn)) + result (transform-fn result) + result-atom (or result-atom (atom nil))] + ;; Don't notify watches now + (set! (.-state result-atom) result) + (add-q! k query inputs result-atom transform-fn query-fn inputs-fn))))))) + + + ;; TODO: Extract several parts to handlers (defn get-current-page @@ -39,7 +178,7 @@ (date/journal-name))] (when page (let [page-name (util/url-decode (string/lower-case page))] - (model/entity (if tag? + (base/entity (if tag? [:tag/name page-name] [:page/name page-name])))))) @@ -106,7 +245,7 @@ (apply concat (for [{:block/keys [ref-pages]} blocks] (map (fn [page] - (when-let [page (model/entity [:page/name (:page/name page)])] + (when-let [page (base/entity [:page/name (:page/name page)])] [:page/refed-blocks (:db/id page)])) ref-pages))) @@ -126,14 +265,14 @@ (filter (fn [v] (and (= (first v) (state/get-current-repo)) (= (second v) :custom))) - (keys @model/query-state)) + (keys @query-state)) (map (fn [v] (vec (drop 1 v))))) block-blocks (some->> (filter (fn [v] (and (= (first v) (state/get-current-repo)) (= (second v) :block/block))) - (keys @model/query-state)) + (keys @query-state)) (map (fn [v] (vec (drop 1 v)))))] (->> @@ -162,7 +301,7 @@ handler-keys (get-handler-keys handler-opts)] (doseq [handler-key handler-keys] (let [handler-key (vec (cons repo-url handler-key))] - (when-let [cache (get @model/query-state handler-key)] + (when-let [cache (get @query-state handler-key)] (let [{:keys [query inputs transform-fn query-fn inputs-fn]} cache] (when (or query query-fn) (let [new-result (-> @@ -177,7 +316,7 @@ (apply d/q query db inputs)) (keyword? query) - (model/get-key-value repo-url query) + (base/get-key-value repo-url query) (seq inputs) (apply d/q query db inputs) @@ -185,7 +324,7 @@ :else (d/q query db)) transform-fn)] - (model/set-new-result! handler-key new-result)))))))))) + (set-new-result! handler-key new-result)))))))))) (catch js/Error e ;; FIXME: check error type and notice user (log/error :db/transact! e))))) @@ -193,9 +332,9 @@ (defn set-key-value [repo-url key value] (if value - (transact-react! repo-url [(model/kv key value)] + (transact-react! repo-url [(kv key value)] {:key [:kv key]}) - (model/remove-key! repo-url key))) + (remove-key! repo-url key))) (defn set-file-content! [repo path content] diff --git a/src/main/frontend/db_mixins.cljs b/src/main/frontend/db_mixins.cljs index 6c49a1d07..71c7faa84 100644 --- a/src/main/frontend/db_mixins.cljs +++ b/src/main/frontend/db_mixins.cljs @@ -1,5 +1,5 @@ (ns frontend.db-mixins - (:require [frontend.db.model :as db])) + (:require [frontend.db.react :as db])) (def query {:wrap-render diff --git a/src/main/frontend/handler/repo.cljs b/src/main/frontend/handler/repo.cljs index 514675e1b..977638379 100644 --- a/src/main/frontend/handler/repo.cljs +++ b/src/main/frontend/handler/repo.cljs @@ -252,6 +252,56 @@ (load-repo-to-db! repo-url {:first-clone? first-clone? :diffs diffs}))) +(defn rebuild-page-blocks-children + "For performance reason, we can update the :block/children value after every operation, + but it's hard to make sure that it's correct, also it needs more time to implement it. + We can improve it if the performance is really an issue." + [repo page] + (let [blocks (->> + (db/get-page-blocks-no-cache repo page {:pull-keys '[:db/id :block/uuid :block/level :block/pre-block? :block/meta]}) + (remove :block/pre-block?) + (map #(select-keys % [:db/id :block/uuid :block/level])) + (reverse)) + original-blocks blocks] + (loop [blocks blocks + tx [] + children {} + last-level 10000] + (if (seq blocks) + (let [[{:block/keys [uuid level] :as block} & others] blocks + [tx children] (cond + (< level last-level) ; parent + (let [cur-children (get children last-level) + tx (if (seq cur-children) + (vec + (concat + tx + (map + (fn [child] + [:db/add (:db/id block) :block/children [:block/uuid child]]) + cur-children))) + tx) + children (-> children + (dissoc last-level) + (update level conj uuid))] + [tx children]) + + (> level last-level) ; child of sibling + (let [children (update children level conj uuid)] + [tx children]) + + :else ; sibling + (let [children (update children last-level conj uuid)] + [tx children]))] + (recur others tx children level)) + ;; TODO: add top-level children to the "Page" block (we might remove the Page from db schema) + (when (seq tx) + (let [delete-tx (map (fn [block] + [:db/retract (:db/id block) :block/children]) + original-blocks)] + (->> (concat delete-tx tx) + (remove nil?)))))))) + (defn transact-react-and-alter-file! [repo tx transact-option files] (spec/validate :repos/url repo)