mirror of https://github.com/logseq/logseq
Performance improvement for inserting new blocks
Tienson Qin
parent
f961d369c1
commit
60b04b0d5d
|
|
@ -17,7 +17,8 @@
|
|||
cljs-drag-n-drop {:mvn/version "0.1.0"}
|
||||
borkdude/sci {:mvn/version "0.1.1-alpha.6"}
|
||||
hickory {:mvn/version "0.7.1"}
|
||||
hiccups {:mvn/version "0.3.0"}}
|
||||
hiccups {:mvn/version "0.3.0"}
|
||||
org.clojure/core.async {:mvn/version "1.3.610"}}
|
||||
|
||||
:aliases {:cljs {:extra-paths ["src/dev-cljs/"]
|
||||
:extra-deps {org.clojure/clojurescript {:mvn/version "1.10.520"}
|
||||
|
|
|
|||
|
|
@ -177,6 +177,9 @@
|
|||
(notification/show! "No changed files yet!" :warning)))))}
|
||||
(fn [e key-code]
|
||||
nil))))
|
||||
{:did-mount (fn [state]
|
||||
(handler/set-save-before-unload!)
|
||||
state)}
|
||||
(mixins/keyboards-mixin keyboards/keyboards)
|
||||
[state route-match main-content]
|
||||
(let [{:keys [open? close-fn open-fn]} state
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@
|
|||
[frontend.util :as util :refer-macros [profile]]
|
||||
[frontend.extensions.sci :as sci]
|
||||
[goog.array :as garray]
|
||||
[frontend.db-schema :as db-schema]))
|
||||
[frontend.db-schema :as db-schema]
|
||||
[clojure.core.async :as async]))
|
||||
|
||||
(defonce brain "🧠")
|
||||
(defonce brain-text "logseq-second-brain")
|
||||
|
|
@ -145,6 +146,10 @@
|
|||
:transform-fn transform-fn})
|
||||
result-atom)
|
||||
|
||||
(defn get-page-blocks-cache-atom
|
||||
[repo page-id]
|
||||
(:result (get @query-state [repo :page/blocks page-id])))
|
||||
|
||||
(defn remove-q!
|
||||
[k]
|
||||
(swap! query-state dissoc k))
|
||||
|
|
@ -480,6 +485,13 @@
|
|||
(group-by-page result)))
|
||||
result)))
|
||||
|
||||
(defn get-repo-tx-id [repo]
|
||||
(when-let [db (get-conn repo)]
|
||||
))
|
||||
|
||||
(defn get-tx-id [tx-report]
|
||||
(get-in tx-report [:tempids :db/current-tx]))
|
||||
|
||||
(defn transact!
|
||||
([tx-data]
|
||||
(transact! (state/get-current-repo) tx-data))
|
||||
|
|
@ -488,7 +500,8 @@
|
|||
(remove nil?))]
|
||||
(when (seq tx-data)
|
||||
(when-let [conn (get-conn repo-url false)]
|
||||
(d/transact! conn (vec tx-data)))))))
|
||||
(let [tx-report (d/transact! conn (vec tx-data))]
|
||||
(state/mark-repo-as-changed! repo-url (get-tx-id tx-report))))))))
|
||||
|
||||
(defn get-key-value
|
||||
([key]
|
||||
|
|
@ -518,6 +531,7 @@
|
|||
(get-conn repo-url false)))]
|
||||
(when (and (seq tx-data) (get-conn))
|
||||
(let [tx-result (profile "Transact!" (d/transact! (get-conn) (vec tx-data)))
|
||||
_ (state/mark-repo-as-changed! repo-url (get-tx-id tx-result))
|
||||
db (:db-after tx-result)
|
||||
handler-keys (get-handler-keys handler-opts)]
|
||||
(doseq [handler-key handler-keys]
|
||||
|
|
@ -1162,16 +1176,16 @@
|
|||
page-list (when-let [list-content (:list directives)]
|
||||
(extract-page-list list-content))]
|
||||
(cond->
|
||||
(util/remove-nils
|
||||
{:page/name (string/lower-case page)
|
||||
:page/original-name page
|
||||
:page/file [:file/path file]
|
||||
:page/journal? journal?
|
||||
:page/journal-day (if journal?
|
||||
(date/journal-title->int (string/capitalize page))
|
||||
0)
|
||||
:page/created-at journal-date-long
|
||||
:page/last-modified-at journal-date-long})
|
||||
(util/remove-nils
|
||||
{:page/name (string/lower-case page)
|
||||
:page/original-name page
|
||||
:page/file [:file/path file]
|
||||
:page/journal? journal?
|
||||
:page/journal-day (if journal?
|
||||
(date/journal-title->int (string/capitalize page))
|
||||
0)
|
||||
:page/created-at journal-date-long
|
||||
:page/last-modified-at journal-date-long})
|
||||
(seq directives)
|
||||
(assoc :page/directives directives)
|
||||
|
||||
|
|
@ -1315,8 +1329,8 @@
|
|||
file-content)
|
||||
tx (concat tx [(let [t (tc/to-long (t/now))]
|
||||
(cond->
|
||||
{:file/path file
|
||||
:file/last-modified-at t}
|
||||
{:file/path file
|
||||
:file/last-modified-at t}
|
||||
new?
|
||||
(assoc :file/created-at t)))])]
|
||||
(transact! repo-url tx))))
|
||||
|
|
@ -1642,20 +1656,18 @@
|
|||
config)))
|
||||
|
||||
(defn start-db-conn!
|
||||
[me repo listen-handler!]
|
||||
[me repo]
|
||||
(let [files-db-name (datascript-files-db repo)
|
||||
files-db-conn (d/create-conn db-schema/files-db-schema)
|
||||
db-name (datascript-db repo)
|
||||
db-conn (d/create-conn db-schema/schema)]
|
||||
(swap! conns assoc files-db-name files-db-conn)
|
||||
(swap! conns assoc db-name db-conn)
|
||||
(when listen-handler!
|
||||
(listen-handler! repo db-conn))
|
||||
(when me
|
||||
(d/transact! db-conn [(me-tx (d/db db-conn) me)]))))
|
||||
|
||||
(defn restore!
|
||||
[{:keys [repos] :as me} listen-handler restore-config-handler]
|
||||
[{:keys [repos] :as me} restore-config-handler]
|
||||
(let [logged? (:name me)]
|
||||
(doall
|
||||
(for [{:keys [url]} repos]
|
||||
|
|
@ -1685,8 +1697,7 @@
|
|||
(reset-conn! db-conn attached-db)))
|
||||
(when logged?
|
||||
(d/transact! db-conn [(me-tx (d/db db-conn) me)])))
|
||||
_ (restore-config-handler repo)]
|
||||
(when stored (listen-handler repo db-conn)))))))))
|
||||
_ (restore-config-handler repo)])))))))
|
||||
|
||||
(defn- build-edges
|
||||
[edges]
|
||||
|
|
@ -1706,13 +1717,13 @@
|
|||
[dark? current-page edges nodes]
|
||||
(mapv (fn [p]
|
||||
(cond->
|
||||
{:id p
|
||||
:name p
|
||||
:val (get-connections p edges)
|
||||
:autoColorBy "group"
|
||||
:group (js/Math.ceil (* (js/Math.random) 12))
|
||||
:color "#222222"
|
||||
}
|
||||
{:id p
|
||||
:name p
|
||||
:val (get-connections p edges)
|
||||
:autoColorBy "group"
|
||||
:group (js/Math.ceil (* (js/Math.random) 12))
|
||||
:color "#222222"
|
||||
}
|
||||
dark?
|
||||
(assoc :color "#8abbbb")
|
||||
(= p current-page)
|
||||
|
|
@ -1850,7 +1861,7 @@
|
|||
|
||||
(defn blocks->vec-tree [col]
|
||||
(let [col (map (fn [h] (cond->
|
||||
h
|
||||
h
|
||||
(not (:block/dummy? h))
|
||||
(dissoc h :block/meta))) col)
|
||||
parent? (fn [item children]
|
||||
|
|
@ -2110,6 +2121,14 @@
|
|||
(and (= typ "Paragraph")
|
||||
(every? #(= % ["Break_Line"]) break-lines))) (rest ast))))))))
|
||||
|
||||
(defn run-batch-txs!
|
||||
[]
|
||||
(let [chan (state/get-db-batch-txs-chan)]
|
||||
(async/go-loop []
|
||||
(let [f (async/<! chan)]
|
||||
(f))
|
||||
(recur))))
|
||||
|
||||
(comment
|
||||
(defn debug!
|
||||
[]
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@
|
|||
(js/clearInterval @interval)
|
||||
(reset! interval nil)
|
||||
(-> (p/all (db/restore! (assoc me :repos repos)
|
||||
repo-handler/db-listen-to-tx!
|
||||
(fn [repo]
|
||||
(file-handler/restore-config! repo false)
|
||||
(when (and (state/logged?)
|
||||
|
|
@ -60,6 +59,35 @@
|
|||
(let [interval-id (js/setInterval inner-fn 50)]
|
||||
(reset! interval interval-id))))
|
||||
|
||||
(defn persist-repo-to-indexeddb!
|
||||
([]
|
||||
(persist-repo-to-indexeddb! false))
|
||||
([force?]
|
||||
(let [status (state/get-repo-persist-status)]
|
||||
(doseq [[repo {:keys [last-stored-at last-modified-at] :as repo-status}] status]
|
||||
(when (and (> last-modified-at last-stored-at)
|
||||
(or force?
|
||||
(and (state/get-edit-input-id)
|
||||
(> (- (util/time-ms) last-stored-at) (* 1 60 1000)) ; 5 minutes
|
||||
)
|
||||
(nil? (state/get-edit-input-id))))
|
||||
(p/let [_ (repo-handler/persist-repo! repo)]
|
||||
(state/update-repo-last-stored-at! repo)))))))
|
||||
|
||||
(defn periodically-persist-repo-to-indexeddb!
|
||||
[]
|
||||
(js/setInterval persist-repo-to-indexeddb! (* 5 1000)))
|
||||
|
||||
(defn set-save-before-unload! []
|
||||
(.addEventListener js/window "beforeunload"
|
||||
(fn [e]
|
||||
(when (state/repos-need-to-be-stored?)
|
||||
;; FIXME: Not working
|
||||
(persist-repo-to-indexeddb! true)
|
||||
(let [message "\\o/"]
|
||||
(set! (.-returnValue (or e js/window.event)) message)
|
||||
message)))))
|
||||
|
||||
(defn start!
|
||||
[render]
|
||||
(let [me (and js/window.user (bean/->clj js/window.user))
|
||||
|
|
@ -76,4 +104,8 @@
|
|||
(notification/show! "Sorry, it seems that your browser doesn't support IndexedDB, we recommend to use latest Chrome(Chromium) or Firefox(Non-private mode)." :error false)
|
||||
(state/set-indexedb-support? false)))
|
||||
|
||||
(restore-and-setup! me repos logged?)))
|
||||
(restore-and-setup! me repos logged?)
|
||||
|
||||
(periodically-persist-repo-to-indexeddb!)
|
||||
|
||||
(db/run-batch-txs!)))
|
||||
|
|
|
|||
|
|
@ -37,7 +37,8 @@
|
|||
*show-block-commands]]
|
||||
[frontend.extensions.html-parser :as html-parser]
|
||||
[medley.core :as medley]
|
||||
[frontend.text :as text]))
|
||||
[frontend.text :as text]
|
||||
[clojure.core.async :as async]))
|
||||
|
||||
;; TODO: refactor the state, it is already too complex.
|
||||
(defonce *last-edit-block (atom nil))
|
||||
|
|
@ -481,22 +482,31 @@
|
|||
value)
|
||||
[new-content value] (new-file-content block file-content value)
|
||||
{:keys [blocks pages start-pos end-pos]} (block/parse-block (assoc block :block/content value) format)
|
||||
first-block (first blocks)
|
||||
last-block (last blocks)
|
||||
blocks (db/recompute-block-children repo block blocks)
|
||||
after-blocks (rebuild-after-blocks repo file (:end-pos meta) end-pos)]
|
||||
(repo-handler/transact-react-and-alter-file!
|
||||
repo
|
||||
(concat
|
||||
pages
|
||||
blocks
|
||||
after-blocks)
|
||||
{:key :block/change
|
||||
:data (map (fn [block] (assoc block :block/page page)) blocks)}
|
||||
[[file-path new-content]])
|
||||
after-blocks (rebuild-after-blocks repo file (:end-pos meta) end-pos)
|
||||
blocks-atom (db/get-page-blocks-cache-atom repo (:db/id page))
|
||||
[before-part after-part] (split-with #(not= (:block/uuid (first blocks)) (:block/uuid %)) @blocks-atom)
|
||||
after-part (rest after-part)]
|
||||
(reset! blocks-atom (->> (concat before-part blocks after-part)
|
||||
(remove nil?)))
|
||||
|
||||
;; Replace with batch transactions
|
||||
(state/add-tx!
|
||||
(fn []
|
||||
(repo-handler/transact-react-and-alter-file!
|
||||
repo
|
||||
(concat
|
||||
pages
|
||||
blocks
|
||||
after-blocks)
|
||||
{:key :block/change
|
||||
:data (map (fn [block] (assoc block :block/page page)) blocks)}
|
||||
[[file-path new-content]])))
|
||||
|
||||
(when ok-handler
|
||||
(ok-handler [first-block last-block v2]))))]
|
||||
(let [first-block (first blocks)
|
||||
last-block (last blocks)]
|
||||
(ok-handler [first-block last-block v2])))))]
|
||||
(cond
|
||||
(and (not file) page)
|
||||
(let [format (name format)
|
||||
|
|
@ -562,20 +572,22 @@
|
|||
:pos pos}))
|
||||
|
||||
(defn edit-block!
|
||||
[block-id prev-pos format id]
|
||||
(let [edit-input-id (str (subs id 0 (- (count id) 36)) block-id)
|
||||
block (or
|
||||
(db/pull [:block/uuid block-id])
|
||||
;; dummy?
|
||||
{:block/uuid block-id
|
||||
:block/content ""})
|
||||
{:block/keys [content]} block
|
||||
content (string/trim (text/remove-level-spaces content format))
|
||||
content-length (count content)
|
||||
text-range (if (or (= :max prev-pos) (<= content-length prev-pos))
|
||||
content
|
||||
(subs content 0 prev-pos))]
|
||||
(state/set-editing! edit-input-id content block text-range)))
|
||||
[block prev-pos format id]
|
||||
(when-let [block-id (:block/uuid block)]
|
||||
(let [edit-input-id (str (subs id 0 (- (count id) 36)) block-id)
|
||||
block (or
|
||||
block
|
||||
(db/pull [:block/uuid block-id])
|
||||
;; dummy?
|
||||
{:block/uuid block-id
|
||||
:block/content ""})
|
||||
{:block/keys [content]} block
|
||||
content (string/trim (text/remove-level-spaces content format))
|
||||
content-length (count content)
|
||||
text-range (if (or (= :max prev-pos) (<= content-length prev-pos))
|
||||
content
|
||||
(subs content 0 prev-pos))]
|
||||
(state/set-editing! edit-input-id content block text-range))))
|
||||
|
||||
(defn- insert-new-block!
|
||||
[state]
|
||||
|
|
@ -598,7 +610,7 @@
|
|||
true
|
||||
(fn [[_first-block last-block _new-block-content]]
|
||||
(let [last-id (:block/uuid last-block)]
|
||||
(edit-block! last-id 0 format id)
|
||||
(edit-block! last-block 0 format id)
|
||||
(clear-when-saved!)))
|
||||
(if last-child true false)
|
||||
(and last-child
|
||||
|
|
@ -734,8 +746,8 @@
|
|||
0)
|
||||
0)]
|
||||
(save-block-if-changed! block new-value)
|
||||
(edit-block! (uuid sibling-block-id)
|
||||
pos format id)))))))))))))
|
||||
(let [block (db/pull (state/get-current-repo) '[*] [:block/uuid (uuid sibling-block-id)])]
|
||||
(edit-block! block pos format id))))))))))))))
|
||||
|
||||
(defn delete-blocks!
|
||||
[repo block-uuids]
|
||||
|
|
@ -1052,7 +1064,8 @@
|
|||
(when (not= (string/trim (text/remove-level-spaces content format))
|
||||
(string/trim value))
|
||||
(save-block! state (:value state))))
|
||||
(edit-block! (uuid sibling-block-id) pos format id)))))))))
|
||||
(let [block (db/pull (state/get-current-repo) '[*] [:block/uuid (uuid sibling-block-id)])]
|
||||
(edit-block! block pos format id))))))))))
|
||||
|
||||
(defn insert-command!
|
||||
[id command-output format {:keys [restore?]
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@
|
|||
last-block (last blocks)]
|
||||
(js/setTimeout
|
||||
#(when-let [first-block (util/get-first-block-by-id (:block/uuid last-block))]
|
||||
(editor-handler/edit-block! (:block/uuid last-block)
|
||||
(editor-handler/edit-block! last-block
|
||||
0
|
||||
(:block/format last-block)
|
||||
(string/replace (gobj/get first-block "id")
|
||||
|
|
|
|||
|
|
@ -263,27 +263,18 @@
|
|||
(create-today-journal-if-not-exists repo-url))
|
||||
(create-contents-file repo-url)))
|
||||
|
||||
(defn db-listen-to-tx!
|
||||
[repo db-conn]
|
||||
(defn persist-repo!
|
||||
[repo]
|
||||
(when-let [files-conn (db/get-files-conn repo)]
|
||||
(d/listen! files-conn :persistence
|
||||
(fn [tx-report]
|
||||
(when (seq (:tx-data tx-report))
|
||||
(when-let [db (:db-after tx-report)]
|
||||
(db/persist repo db true))))))
|
||||
(d/listen! db-conn :persistence
|
||||
(fn [tx-report]
|
||||
(when (seq (:tx-data tx-report))
|
||||
(when-let [db (:db-after tx-report)]
|
||||
(db/persist repo db false))))))
|
||||
(db/persist repo @files-conn true))
|
||||
(when-let [db (db/get-conn repo)]
|
||||
(prn "persisted core db")
|
||||
(db/persist repo db false)))
|
||||
|
||||
(defn load-db-and-journals!
|
||||
[repo-url diffs first-clone?]
|
||||
(when (or diffs first-clone?)
|
||||
(p/let [_ (load-repo-to-db! repo-url diffs first-clone?)]
|
||||
(when-let [conn (db/get-conn repo-url false)]
|
||||
(db-listen-to-tx! repo-url conn))
|
||||
|
||||
(when first-clone?
|
||||
(create-default-files! repo-url))
|
||||
|
||||
|
|
@ -468,7 +459,7 @@
|
|||
(fn [result]
|
||||
(state/set-git-clone-repo! "")
|
||||
(state/set-current-repo! repo-url)
|
||||
(db/start-db-conn! (:me @state/state) repo-url nil)
|
||||
(db/start-db-conn! (:me @state/state) repo-url)
|
||||
(db/mark-repo-as-cloned repo-url)
|
||||
(git-handler/set-latest-commit-if-exists! repo-url))
|
||||
(fn [e]
|
||||
|
|
@ -519,7 +510,8 @@
|
|||
(db/remove-files-db! url)
|
||||
(fs/rmdir (util/get-repo-dir url))
|
||||
(state/delete-repo! repo)
|
||||
(state/clear-changed-files! repo))
|
||||
(state/clear-changed-files! repo)
|
||||
)
|
||||
(fn [error]
|
||||
(prn "Delete repo failed, error: " error))))
|
||||
|
||||
|
|
@ -530,7 +522,7 @@
|
|||
(p/let [result (-> (fs/mkdir (str "/" repo))
|
||||
(p/catch (fn [_e] nil)))
|
||||
_ (state/set-current-repo! repo)
|
||||
_ (db/start-db-conn! nil repo db-listen-to-tx!)
|
||||
_ (db/start-db-conn! nil repo)
|
||||
_ (create-month-journal-if-not-exists repo)
|
||||
_ (create-config-file-if-not-exists repo)
|
||||
_ (create-contents-file repo)]
|
||||
|
|
|
|||
|
|
@ -6,12 +6,16 @@
|
|||
[medley.core :as medley]
|
||||
[goog.object :as gobj]
|
||||
[goog.dom :as gdom]
|
||||
[dommy.core :as dom]))
|
||||
[dommy.core :as dom]
|
||||
[cljs-time.core :as t]
|
||||
[cljs-time.coerce :as tc]
|
||||
[clojure.core.async :as async]))
|
||||
|
||||
(defonce state
|
||||
(atom
|
||||
{:route-match nil
|
||||
:today nil
|
||||
:db/batch-txs (async/chan 100)
|
||||
:notification/show? false
|
||||
:notification/content nil
|
||||
:repo/cloning? false
|
||||
|
|
@ -22,6 +26,11 @@
|
|||
(storage/get "git-changed-files")
|
||||
{})
|
||||
:indexeddb/support? true
|
||||
;; TODO: save in local storage so that if :changed? is true when user
|
||||
;; reloads the browser, the app should re-index the repo (another way
|
||||
;; is to save all the tx data since :last-stored-at)
|
||||
;; repo -> {:last-stored-at :last-modified-at}
|
||||
:repo/persist-status {}
|
||||
:me nil
|
||||
:git/clone-repo (or (storage/get :git/clone-repo) "")
|
||||
:git/current-repo (storage/get :git/current-repo)
|
||||
|
|
@ -413,9 +422,9 @@
|
|||
repos (get-in @state [:me :repos])]
|
||||
(when (seq repos)
|
||||
(let [repos (mapv (fn [{:keys [installation_id] :as r}]
|
||||
(if-let [token (get tokens installation_id)]
|
||||
(assoc r :token token)
|
||||
r)) repos)]
|
||||
(if-let [token (get tokens installation_id)]
|
||||
(assoc r :token token)
|
||||
r)) repos)]
|
||||
(swap! state assoc-in [:me :repos] repos))))))
|
||||
|
||||
(defn get-github-token
|
||||
|
|
@ -651,3 +660,34 @@
|
|||
(when-let [basis (get-in @state [:config repo :journal-basis])]
|
||||
(keyword (string/lower-case (str basis)))))
|
||||
:monthly))
|
||||
|
||||
(defn update-repo-last-stored-at!
|
||||
[repo]
|
||||
(swap! state assoc-in [:repo/persist-status repo :last-stored-at] (util/time-ms)))
|
||||
|
||||
(defn get-repo-persist-status
|
||||
[]
|
||||
(:repo/persist-status @state))
|
||||
|
||||
(defn mark-repo-as-changed!
|
||||
[repo _tx-id]
|
||||
(swap! state assoc-in [:repo/persist-status repo :last-modified-at] (util/time-ms)))
|
||||
|
||||
(defn add-tx!
|
||||
;; TODO: replace f with data for batch transactions
|
||||
[f]
|
||||
(when f
|
||||
(swap! state update :db/batch-txs (fn [chan]
|
||||
(async/put! chan f)
|
||||
chan))))
|
||||
|
||||
(defn get-db-batch-txs-chan
|
||||
[]
|
||||
(:db/batch-txs @state))
|
||||
|
||||
(defn repos-need-to-be-stored?
|
||||
[]
|
||||
(let [status (vals (get-repo-persist-status))]
|
||||
(some (fn [{:keys [last-stored-at last-modified-at]}]
|
||||
(> last-modified-at last-stored-at))
|
||||
status)))
|
||||
|
|
|
|||
Loading…
Reference in New Issue