greatbody
/
logseq
mirror of https://github.com/logseq/logseq
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Both load && persist work now

feat/datascript-storage-test
Tienson Qin 2023-12-07 23:15:04 +08:00
parent 2503ceb720
commit ba2aaae000
6 changed files with 133 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/frontend/db/persist.cljs
View File

@ -14,7 +14,7 @@
(p/let [repos (idb/get-nfs-dbs) (p/let [repos (idb/get-nfs-dbs)
db-repos (persist-db/<list-db) db-repos (persist-db/<list-db)
electron-disk-graphs (when (util/electron?) (ipc/ipc "getGraphs"))] electron-disk-graphs (when (util/electron?) (ipc/ipc "getGraphs"))]
(distinct (concat repos db-repos (bean/->clj electron-disk-graphs))))) (distinct (concat repos db-repos (some-> electron-disk-graphs bean/->clj)))))
(defn get-serialized-graph (defn get-serialized-graph
[graph-name] [graph-name]

73
src/main/frontend/db_worker.cljs
View File

@ -14,6 +14,16 @@
(defonce *sqlite (atom nil)) (defonce *sqlite (atom nil))
(defonce *sqlite-db (atom nil)) (defonce *sqlite-db (atom nil))
(defonce *datascript-conn (atom nil)) (defonce *datascript-conn (atom nil))
(defonce *opfs-pool (atom nil))
(defn- get-opfs-pool
[]
(or @*opfs-pool
(p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name "logseq-db"
:initialCapacity 100})]
;; (.removeVfs pool)
(reset! *opfs-pool pool)
pool)))
(defn- init-sqlite-module! (defn- init-sqlite-module!
[] []
@ -23,11 +33,25 @@
sqlite (sqlite3InitModule (clj->js {:url sqlite-wasm-url sqlite (sqlite3InitModule (clj->js {:url sqlite-wasm-url
:print js/console.log :print js/console.log
:printErr js/console.error}))] :printErr js/console.error}))]
(reset! *sqlite sqlite)))) (reset! *sqlite sqlite)
nil)))
(defn- close-all-dbs! (defn- remove-pfs!
"!! use it only for development"
[] []
) (when-let [^js pool (get-opfs-pool)]
(.removeVfs ^js pool)))
(defn- get-file-names
[]
(when-let [^js pool (get-opfs-pool)]
(.getFileNames pool)))
(defn- export-db-file
[file-path]
;; TODO: get file name by repo
(when-let [^js pool (get-opfs-pool)]
(.exportFile ^js pool file-path)))
(defn upsert-addr-content! (defn upsert-addr-content!
"Upsert addr+data-seq" "Upsert addr+data-seq"
@ -62,8 +86,21 @@
(upsert-addr-content! data))) (upsert-addr-content! data)))
(-restore [_ addr] (-restore [_ addr]
(let [content (restore-data-from-addr addr)] (restore-data-from-addr addr)))))
(edn/read-string content))))))
(defn- create-or-open-db!
[repo]
(p/let [pool (get-opfs-pool)
db (new (.-OpfsSAHPoolDb pool) (str "/" repo ".sqlite"))
storage (new-sqlite-storage repo {})]
(js/console.dir db)
(reset! *sqlite-db db)
(.exec db "PRAGMA locking_mode=exclusive")
(.exec db "create table if not exists kvs (addr INTEGER primary key, content TEXT)")
(let [conn (or (d/restore-conn storage)
(d/create-conn db-schema/schema-for-db-based-graph {:storage storage}))]
(reset! *datascript-conn conn)
nil)))
#_:clj-kondo/ignore #_:clj-kondo/ignore
(defclass SQLiteDB (defclass SQLiteDB
@ -90,8 +127,7 @@
(init (init
[_this] [_this]
(init-sqlite-module!) (init-sqlite-module!))
nil)
(inited (inited
[_this] [_this]
@ -99,24 +135,14 @@
(listDB (listDB
[_this] [_this]
;; (.list_db sqlite-db) ;; TODO:
#js []) (prn (get-file-names))
nil)
(newDB (createOrOpenDB
[_this repo] [_this repo]
;; TODO: close all the other db connections ;; TODO: close all the other db connections
(p/let [sqlite @*sqlite (create-or-open-db! repo))
db-name repo
pool (.installOpfsSAHPoolVfs sqlite #js {:name db-name})
db (new (.-OpfsSAHPoolDb pool) "/logseq")
storage (new-sqlite-storage db-name {})]
(reset! *sqlite-db db)
(.exec db "PRAGMA locking_mode=exclusive")
(.exec db "create table if not exists kvs (addr INTEGER primary key, content TEXT)")
(let [conn (or (d/restore-conn storage)
(d/create-conn db-schema/schema-for-db-based-graph {:storage storage}))]
(reset! *datascript-conn conn)
nil)))
(transact (transact
[_this repo tx-data tx-meta] [_this repo tx-data tx-meta]
@ -142,5 +168,4 @@
"web worker entry" "web worker entry"
[] []
(let [^js obj (SQLiteDB.)] (let [^js obj (SQLiteDB.)]
(p/let [_ (init-sqlite-module!)] (Comlink/expose obj)))
(Comlink/expose obj))))

127
src/main/frontend/handler.cljs
View File

@ -75,58 +75,52 @@
(f) (f)
(js/setInterval f 5000))) (js/setInterval f 5000)))
(defn- instrument!
[]
(let [total (srs/get-srs-cards-total)]
(state/set-state! :srs/cards-due-count total)))
(defn restore-and-setup! (defn restore-and-setup!
[repos] [repo repos]
(when-let [repo (or (state/get-current-repo) (:url (first repos)))] (-> (p/do!
(-> (p/do! (db-restore/restore-graph! repo)
(db-restore/restore-graph! repo) (repo-config-handler/start {:repo repo})
(repo-config-handler/start {:repo repo}) (op-mem-layer/<init-load-from-indexeddb! repo))
(op-mem-layer/<init-load-from-indexeddb! repo)) (p/then
(p/then (fn []
(fn [] (db-listener/listen-and-persist! repo)
(db-listener/listen-and-persist! repo)
;; try to load custom css only for current repo ;; try to load custom css only for current repo
(ui-handler/add-style-if-exists!) (ui-handler/add-style-if-exists!)
(-> (->
(p/do! (p/do!
(when (config/global-config-enabled?) (when (config/global-config-enabled?)
(global-config-handler/start {:repo repo})) (global-config-handler/start {:repo repo}))
(when (config/plugin-config-enabled?) (when (config/plugin-config-enabled?)
(plugin-config-handler/start))) (plugin-config-handler/start)))
(p/finally (p/finally
(fn [] (fn []
;; install after config is restored ;; install after config is restored
(shortcut/refresh!) (shortcut/refresh!)
(cond (cond
(and (not (seq (db/get-files config/local-repo))) (and (not (seq (db/get-files config/local-repo)))
;; Not native local directory ;; Not native local directory
(not (some config/local-file-based-graph? (map :url repos))) (not (some config/local-file-based-graph? (map :url repos)))
(not (mobile-util/native-platform?))) (not (mobile-util/native-platform?)))
;; will execute `(state/set-db-restoring! false)` inside ;; will execute `(state/set-db-restoring! false)` inside
(repo-handler/setup-local-repo-if-not-exists!) (repo-handler/setup-local-repo-if-not-exists!)
:else :else
(state/set-db-restoring! false))))))) (state/set-db-restoring! false)))))))
(p/then (p/then
(fn [] (fn []
(js/console.log "db restored, setting up repo hooks") (js/console.log "db restored, setting up repo hooks")
(state/pub-event! [:modal/nfs-ask-permission]) (state/pub-event! [:modal/nfs-ask-permission])
(page-handler/init-commands!) (page-handler/init-commands!)
(watch-for-date!) (watch-for-date!)
(file-handler/watch-for-current-graph-dir!) (file-handler/watch-for-current-graph-dir!)
(state/pub-event! [:graph/restored (state/get-current-repo)]))) (state/pub-event! [:graph/restored (state/get-current-repo)])))
(p/catch (fn [error] (p/catch (fn [error]
(log/error :exception error)))))) (log/error :exception error)))))
(defn- handle-connection-change (defn- handle-connection-change
[e] [e]
@ -169,27 +163,30 @@
;; FIXME: Another get-repos implementation at src\main\frontend\handler\repo.cljs ;; FIXME: Another get-repos implementation at src\main\frontend\handler\repo.cljs
(defn- get-repos (defn- get-repos
[] []
(p/let [nfs-dbs (db-persist/get-all-graphs)] (->
;; TODO: Better IndexDB migration handling (p/let [nfs-dbs (db-persist/get-all-graphs)]
(cond ;; TODO: Better IndexDB migration handling
(and (mobile-util/native-platform?) (cond
(some #(or (string/includes? % " ") (and (mobile-util/native-platform?)
(string/includes? % "logseq_local_/")) nfs-dbs)) (some #(or (string/includes? % " ")
(do (notification/show! ["DB version is not compatible, please clear cache then re-add your graph back." (string/includes? % "logseq_local_/")) nfs-dbs))
(ui/button (do (notification/show! ["DB version is not compatible, please clear cache then re-add your graph back."
(t :settings-page/clear-cache) (ui/button
:class "ui__modal-enter" (t :settings-page/clear-cache)
:class "text-sm p-1" :class "ui__modal-enter"
:on-click clear-cache!)] :error false) :class "text-sm p-1"
{:url config/local-repo :on-click clear-cache!)] :error false)
:example? true}) {:url config/local-repo
:example? true})
(seq nfs-dbs) (seq nfs-dbs)
(map (fn [db] {:url db :nfs? true}) nfs-dbs) (map (fn [db] {:url db :nfs? true}) nfs-dbs)
:else :else
[{:url config/local-repo [{:url config/local-repo
:example? true}]))) :example? true}]))
(p/catch (fn [error]
(js/console.error)))))
(defn- register-components-fns! (defn- register-components-fns!
[] []
@ -214,7 +211,6 @@
[render] [render]
(test/setup-test!) (test/setup-test!)
(get-system-info) (get-system-info)
(db-browser/start-db-worker!)
(set-global-error-notification!) (set-global-error-notification!)
(set! js/window.onhashchange #(state/hide-custom-context-menu!)) ;; close context menu when page navs (set! js/window.onhashchange #(state/hide-custom-context-menu!)) ;; close context menu when page navs
@ -244,10 +240,12 @@
(p/do! (p/do!
(when (mobile-util/native-platform?) (when (mobile-util/native-platform?)
(mobile/mobile-preinit)) (mobile/mobile-preinit))
(-> (p/let [repos (get-repos) (-> (p/let [_ (db-browser/start-db-worker!)
repos (get-repos)
_ (state/set-repos! repos) _ (state/set-repos! repos)
_ (mobile-util/hide-splash) ;; hide splash as early as ui is stable _ (mobile-util/hide-splash) ;; hide splash as early as ui is stable
_ (restore-and-setup! repos)] repo (or (state/get-current-repo) (:url (first repos)))
_ (restore-and-setup! repo repos)]
(when (mobile-util/native-platform?) (when (mobile-util/native-platform?)
(state/restore-mobile-theme!))) (state/restore-mobile-theme!)))
(p/catch (fn [e] (p/catch (fn [e]
@ -260,8 +258,7 @@
(when config/dev? (when config/dev?
(enable-datalog-console)) (enable-datalog-console))
(persist-var/load-vars) (persist-var/load-vars)))
(js/setTimeout instrument! (* 60 1000))))
(defn stop! [] (defn stop! []
(prn "stop!")) (prn "stop!"))

4
src/main/frontend/persist_db.cljs
View File

@ -29,6 +29,6 @@
([repo] ([repo]
(<fetch-init-data repo {})) (<fetch-init-data repo {}))
([repo opts] ([repo opts]
(p/let [ret (protocol/<fetch-initital-data (get-impl) repo opts)] (p/let [ret (protocol/<fetch-initial-data (get-impl) repo opts)]
(js/console.log "fetch-initital" ret) (js/console.log "fetch-initial-data" ret)
ret))) ret)))

38
src/main/frontend/persist_db/browser.cljs
View File

@ -8,7 +8,8 @@
[frontend.config :as config] [frontend.config :as config]
[promesa.core :as p] [promesa.core :as p]
[frontend.util :as util] [frontend.util :as util]
[frontend.handler.notification :as notification])) [frontend.handler.notification :as notification]
[cljs-bean.core :as bean]))
(defonce *sqlite (atom nil)) (defonce *sqlite (atom nil))
@ -20,16 +21,16 @@
"/static/js/db-worker.js") "/static/js/db-worker.js")
worker (js/Worker. worker-url) worker (js/Worker. worker-url)
sqlite (Comlink/wrap worker)] sqlite (Comlink/wrap worker)]
(reset! *sqlite sqlite)))) (reset! *sqlite sqlite)
(.init sqlite))))
(defrecord InBrowser [] (defrecord InBrowser []
protocol/PersistentDB protocol/PersistentDB
(<new [_this repo] (<new [_this repo]
(prn :debug ::new-repo repo)
(let [^js sqlite @*sqlite] (let [^js sqlite @*sqlite]
(-> (.newDB sqlite repo) (-> (.createOrOpenDB sqlite repo)
(p/then (fn [result] (p/then (fn [_result]
(prn "SQLite db created successfully: " repo))) (prn "SQLite db created or opened successfully: " repo)))
(p/catch (fn [error] (p/catch (fn [error]
(js/console.error error) (js/console.error error)
(notification/show! [:div (str "SQLiteDB creation error: " error)] :error) (notification/show! [:div (str "SQLiteDB creation error: " error)] :error)
@ -38,6 +39,8 @@
(<list-db [_this] (<list-db [_this]
(when-let [^js sqlite @*sqlite] (when-let [^js sqlite @*sqlite]
(-> (.listDB sqlite) (-> (.listDB sqlite)
(p/then (fn [result]
(bean/->clj result)))
(p/catch (fn [error] (p/catch (fn [error]
(prn :debug :list-db-error (js/Date.)) (prn :debug :list-db-error (js/Date.))
(notification/show! [:div (str "SQLiteDB error: " error)] :error) (notification/show! [:div (str "SQLiteDB error: " error)] :error)
@ -50,21 +53,16 @@
) )
(<transact-data [_this repo tx-data tx-meta] (<transact-data [_this repo tx-data tx-meta]
(prn :debug ::transact-data repo (count tx-data) (count tx-meta)) (when-let [^js sqlite @*sqlite]
(let [^js sqlite @*sqlite]
(p->c (p->c
(p/let [_ (.transact sqlite repo (pr-str tx-data) (pr-str tx-meta))] (p/let [_ (.transact sqlite repo (pr-str tx-data) (pr-str tx-meta))]
nil)))) nil))))
(<fetch-initital-data [_this repo _opts] (<fetch-initial-data [_this repo _opts]
(prn ::fetch-initital-data repo @*sqlite) (when-let [^js sqlite @*sqlite]
(-> (let [^js sqlite @*sqlite (-> (p/let [_ (.createOrOpenDB sqlite repo)]
;; <fetch-initital-data is called when init/re-loading graph (.getInitialData sqlite repo))
;; the underlying DB should be opened (p/catch (fn [error]
] (prn :debug :fetch-initial-data-error)
(p/let [_ (.newDB sqlite repo)] (js/console.error error)
(.getInitialData sqlite repo))) (notification/show! [:div (str "SQLiteDB fetch error: " error)] :error) {}))))))
(p/catch (fn [error]
(prn :debug :fetch-initial-data-error)
(js/console.error error)
(notification/show! [:div (str "SQLiteDB fetch error: " error)] :error) {})))))

2
src/main/frontend/persist_db/protocol.cljs
View File

@ -8,4 +8,4 @@
(<new [this repo]) (<new [this repo])
(<unsafe-delete [this repo]) (<unsafe-delete [this repo])
(<transact-data [this repo tx-data tx-meta] "Transact data to db") (<transact-data [this repo tx-data tx-meta] "Transact data to db")
(<fetch-initital-data [this repo opts])) (<fetch-initial-data [this repo opts]))