feat/datascript-storage-test
Tienson Qin 2023-11-25 14:14:02 +08:00
parent a408f93b26
commit a4c30bc62f
3 changed files with 22 additions and 52 deletions

View File

@ -9,7 +9,8 @@
[cljs-bean.core :as bean] [cljs-bean.core :as bean]
[cljs.cache :as cache] [cljs.cache :as cache]
[datascript.core :as d] [datascript.core :as d]
[goog.object :as gobj])) [goog.object :as gobj]
[logseq.db.frontend.schema :as db-schema]))
(defn- write-transit [data] (defn- write-transit [data]
(t/write (t/writer :json) data)) (t/write (t/writer :json) data))
@ -74,41 +75,6 @@
graph-dir (node-path/join graphs-dir db-name')] graph-dir (node-path/join graphs-dir db-name')]
[db-name' (node-path/join graph-dir "db.sqlite")])) [db-name' (node-path/join graph-dir "db.sqlite")]))
(defn- clj-list->sql
"Turn clojure list into SQL list
'(1 2 3 4)
->
\"('1','2','3','4')\""
[ids]
(str "(" (->> (map (fn [id] (str "'" id "'")) ids)
(string/join ", ")) ")"))
(defn upsert-blocks!
"Creates or updates given js blocks"
[repo blocks]
(when-let [db (get-db repo)]
(let [insert (prepare db "INSERT INTO blocks (uuid, type, page_uuid, page_journal_day, name, content,datoms, created_at, updated_at) VALUES (@uuid, @type, @page_uuid, @page_journal_day, @name, @content, @datoms, @created_at, @updated_at) ON CONFLICT (uuid) DO UPDATE SET (type, page_uuid, page_journal_day, name, content, datoms, created_at, updated_at) = (@type, @page_uuid, @page_journal_day, @name, @content, @datoms, @created_at, @updated_at)"
repo)
insert-many (.transaction ^object db
(fn [blocks]
(doseq [block blocks]
(.run ^object insert block))))]
(insert-many blocks))))
(defn delete-blocks!
[repo uuids]
(when-let [db (get-db repo)]
(let [sql (str "DELETE from blocks WHERE uuid IN " (clj-list->sql uuids))
stmt (prepare db sql repo)]
(.run ^object stmt))))
;; Initial data:
;; All pages and block ids
;; latest 3 journals
;; other data such as config.edn, custom css/js
;; current page, sidebar blocks
(defn query (defn query
[repo db sql] [repo db sql]
(let [stmt (prepare db sql repo)] (let [stmt (prepare db sql repo)]
@ -128,11 +94,12 @@
(defn restore-data-from-addr (defn restore-data-from-addr
[repo addr] [repo addr]
(when-let [db (get-db repo)] (when addr
(-> (query repo db (when-let [db (get-db repo)]
(str "select content from kvs where addr = " addr)) (-> (query repo db
first (str "select content from kvs where addr = " addr))
(gobj/get "content")))) first
(gobj/get "content")))))
(defn sqlite-storage (defn sqlite-storage
[repo {:keys [threshold] [repo {:keys [threshold]
@ -166,16 +133,18 @@
(swap! databases assoc db-sanitized-name db) (swap! databases assoc db-sanitized-name db)
(let [storage (sqlite-storage db-name {}) (let [storage (sqlite-storage db-name {})
conn (or (d/restore-conn storage) conn (or (d/restore-conn storage)
(d/create-conn nil {:storage storage}))] (d/create-conn db-schema/schema-for-db-based-graph {:storage storage}))]
(swap! conns assoc db-name conn))) (swap! conns assoc db-sanitized-name conn)))
nil) nil)
(defn transact! (defn transact!
[repo tx-data tx-meta] [repo tx-data tx-meta]
(prn :transit {:tx-data tx-data
:tx-meta tx-meta})
(when-let [conn (get-conn repo)] (when-let [conn (get-conn repo)]
(d/transact! conn tx-data tx-meta))) (try
(d/transact! conn tx-data tx-meta)
(catch :default e
(prn :debug :error)
(js/console.error e)))))
(defn get-initial-data (defn get-initial-data
"Get all datoms remove :block/content" "Get all datoms remove :block/content"
@ -184,4 +153,4 @@
(let [db @conn] (let [db @conn]
(->> (d/datoms db :eavt) (->> (d/datoms db :eavt)
;; (remove (fn [e] (= :block/content (:a e)))) ;; (remove (fn [e] (= :block/content (:a e))))
)))) vec))))

View File

@ -34,7 +34,8 @@
[logseq.db.sqlite.db :as sqlite-db] [logseq.db.sqlite.db :as sqlite-db]
[logseq.db.sqlite.util :as sqlite-util] [logseq.db.sqlite.util :as sqlite-util]
[logseq.common.graph :as common-graph] [logseq.common.graph :as common-graph]
[promesa.core :as p])) [promesa.core :as p]
[datascript.transit :as dt]))
(defmethod handle :mkdir [_window [_ dir]] (defmethod handle :mkdir [_window [_ dir]]
(fs/mkdirSync dir)) (fs/mkdirSync dir))
@ -375,7 +376,6 @@
(db/new-db! repo)) (db/new-db! repo))
(defmethod handle :db-transact-data [_window [_ repo data-str]] (defmethod handle :db-transact-data [_window [_ repo data-str]]
(prn :debug :data-str data-str)
(let [{:keys [tx-data tx-meta]} (reader/read-string data-str)] (let [{:keys [tx-data tx-meta]} (reader/read-string data-str)]
(sqlite-db/transact! repo tx-data tx-meta) (sqlite-db/transact! repo tx-data tx-meta)
nil)) nil))
@ -383,7 +383,7 @@
;; Needs to be called first for an existing graph ;; Needs to be called first for an existing graph
(defmethod handle :get-initial-data [_window [_ repo _opts]] (defmethod handle :get-initial-data [_window [_ repo _opts]]
(db/open-db! repo) (db/open-db! repo)
(sqlite-db/get-initial-data repo)) (dt/write-transit-str (sqlite-db/get-initial-data repo)))
(defmethod handle :get-other-data [_window [_ repo journal-block-uuids _opts]] (defmethod handle :get-other-data [_window [_ repo journal-block-uuids _opts]]
nil) nil)

View File

@ -18,7 +18,8 @@
[frontend.util :as util] [frontend.util :as util]
[cljs-time.core :as t] [cljs-time.core :as t]
[logseq.db.frontend.property :as db-property] [logseq.db.frontend.property :as db-property]
[cljs-bean.core :as bean])) [cljs-bean.core :as bean]
[datascript.transit :as dt]))
(defn- old-schema? (defn- old-schema?
"Requires migration if the schema version is older than db-schema/version" "Requires migration if the schema version is older than db-schema/version"
@ -127,7 +128,7 @@
(fn profiled-d-conn [& args] (fn profiled-d-conn [& args]
(util/profile :restore-graph-from-sqlite!-init-db (apply d/conn-from-datoms args)))})) (util/profile :restore-graph-from-sqlite!-init-db (apply d/conn-from-datoms args)))}))
[conn datoms-count] (if electron? [conn datoms-count] (if electron?
(let [datoms (bean/->clj data)] (let [datoms (dt/read-transit-str (or data ""))]
[(d/conn-from-datoms datoms db-schema/schema-for-db-based-graph) [(d/conn-from-datoms datoms db-schema/schema-for-db-based-graph)
(count datoms)]) (count datoms)])
[conn datoms-count]) [conn datoms-count])